Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(webhooks): process webhook consumer events and send items to webhooks (#4710)
Will Hutchinson74a026af7926
diff --git a/docker-compose.local-e2e.yml b/docker-compose.local-e2e.ymlindex c415b5dfc7..481f82ebb7 100644--- a/docker-compose.local-e2e.yml+++ b/docker-compose.local-e2e.yml@@ -43,6 +43,7 @@ x-local-e2e-env: &local-e2e-envOVERRIDE_NOTIFICATION_QUEUE: http://localstack:4566/000000000000/notification-queueOVERRIDE_NOTIFICATION_INGRESS_QUEUE: http://localstack:4566/000000000000/notification-ingress-queueOVERRIDE_PUSH_NOTIFICATION_EVENT_HANDLER_QUEUE: push-delivery-queue+ OVERRIDE_WEBHOOK_EVENT_QUEUE: http://localstack:4566/000000000000/webhook-event-queue.fifoOVERRIDE_SEARCH_EVENT_QUEUE: search-event-queueOVERRIDE_AI_PROJECTION_QUEUE: http://localstack:4566/000000000000/ai-projection-queueOVERRIDE_SFS_DELETE_QUEUE: email-sfs-delete-queuediff --git a/infra/stacks/cloud-storage-service/index.ts b/infra/stacks/cloud-storage-service/index.tsindex 12b134694a..bab7dcc820 100644--- a/infra/stacks/cloud-storage-service/index.ts+++ b/infra/stacks/cloud-storage-service/index.ts@@ -1,6 +1,6 @@import * as aws from '@pulumi/aws';import * as pulumi from '@pulumi/pulumi';-import { createBucket } from '../../packages/resources';+import { createBucket, Queue } from '../../packages/resources';import {config,getMacroApiToken,@@ -30,6 +30,16 @@ const tags = {project: 'cloud-storage-service',};+const webhookEventQueue = new Queue('webhook-event', {+ fifoQueue: true,+ maxReceiveCount: 20,+ tags,+ visibilityTimeoutSeconds: 30,+});++export const webhookEventQueueArn = webhookEventQueue.queue.arn;+export const webhookEventQueueName = webhookEventQueue.queue.name;+const MACRO_CACHE = aws.secretsmanager.getSecretVersionOutput({secretId: config.require(`macro_cache_secret_key`),@@ -246,6 +256,7 @@ const cloudStorageService = new CloudStorageService(cloudStorageClusterName: cloudStorageClusterName,queueArns: [searchEventQueueArn,+ webhookEventQueue.queue.arn,deleteDocumentHandler.queue.arn,notificationIngressQueueArn,contactsQueueArn,diff --git a/js/app/packages/service-clients/service-storage/generated/schemas/createWebhookRequest.ts b/js/app/packages/service-clients/service-storage/generated/schemas/createWebhookRequest.tsindex a6f79b6eb7..6070a15a3e 100644--- a/js/app/packages/service-clients/service-storage/generated/schemas/createWebhookRequest.ts+++ b/js/app/packages/service-clients/service-storage/generated/schemas/createWebhookRequest.ts@@ -13,7 +13,7 @@ import type { WebhookScope } from './webhookScope';* Request to create a webhook.*/export interface CreateWebhookRequest {- /** HTTPS endpoint URL. */+ /** Endpoint URL. HTTPS is required outside local environments. */endpoint_url: string;/** Typed filters used to match events and optional entity ids. */filters: Vec;diff --git a/js/app/packages/service-clients/service-storage/generated/schemas/patchWebhookRequest.ts b/js/app/packages/service-clients/service-storage/generated/schemas/patchWebhookRequest.tsindex d6fb1467e2..61f1555f2e 100644--- a/js/app/packages/service-clients/service-storage/generated/schemas/patchWebhookRequest.ts+++ b/js/app/packages/service-clients/service-storage/generated/schemas/patchWebhookRequest.ts@@ -14,7 +14,7 @@ import type { PatchWebhookRequestStatus } from './patchWebhookRequestStatus';* Request to patch a webhook.*/export interface PatchWebhookRequest {- /** HTTPS endpoint URL. */+ /** Endpoint URL. HTTPS is required outside local environments. */endpoint_url?: PatchWebhookRequestEndpointUrl;filters?: PatchWebhookRequestFilters;headers?: PatchWebhookRequestHeaders;diff --git a/js/app/packages/service-clients/service-storage/generated/schemas/patchWebhookRequestEndpointUrl.ts b/js/app/packages/service-clients/service-storage/generated/schemas/patchWebhookRequestEndpointUrl.tsindex 6aa42093a7..f6c13caa1e 100644--- a/js/app/packages/service-clients/service-storage/generated/schemas/patchWebhookRequestEndpointUrl.ts+++ b/js/app/packages/service-clients/service-storage/generated/schemas/patchWebhookRequestEndpointUrl.ts@@ -6,6 +6,6 @@*//**- * HTTPS endpoint URL.+ * Endpoint URL. HTTPS is required outside local environments.*/export type PatchWebhookRequestEndpointUrl = string | null;diff --git a/js/app/packages/service-clients/service-storage/generated/schemas/webhook.ts b/js/app/packages/service-clients/service-storage/generated/schemas/webhook.tsindex 57bf5efc13..7b1f50072e 100644--- a/js/app/packages/service-clients/service-storage/generated/schemas/webhook.ts+++ b/js/app/packages/service-clients/service-storage/generated/schemas/webhook.ts@@ -21,7 +21,7 @@ export interface Webhook {created_by_user_id: string;/** Soft-delete timestamp. */deleted_at?: WebhookDeletedAt;- /** HTTPS endpoint URL. */+ /** Endpoint URL. HTTPS is required outside local environments. */endpoint_url: string;/** Typed filters used to match events and optional entity ids. */filters: Vec;diff --git a/js/app/packages/service-clients/service-storage/generated/zod.ts b/js/app/packages/service-clients/service-storage/generated/zod.tsindex 5703c264ba..7718dc353c 100644--- a/js/app/packages/service-clients/service-storage/generated/zod.ts+++ b/js/app/packages/service-clients/service-storage/generated/zod.ts@@ -20029,7 +20029,9 @@ export const editProjectV2Response = zod.object({*/export const createWebhookBody = zod.object({- endpoint_url: zod.string().describe('HTTPS endpoint URL.'),+ endpoint_url: zod+ .string()+ .describe('Endpoint URL. HTTPS is required outside local environments.'),filters: zod.array(zod.object({@@ -20073,7 +20075,10 @@ export const patchWebhookParams = zod.object({export const patchWebhookBody = zod.object({- endpoint_url: zod.string().nullish().describe('HTTPS endpoint URL.'),+ endpoint_url: zod+ .string()+ .nullish()+ .describe('Endpoint URL. HTTPS is required outside local environments.'),filters: zod.union([zod.null(),@@ -20119,7 +20124,9 @@ export const patchWebhookResponse = zod.datetime({}).nullish().describe('Soft-delete timestamp.'),- endpoint_url: zod.string().describe('HTTPS endpoint URL.'),+ endpoint_url: zod+ .string()+ .describe('Endpoint URL. HTTPS is required outside local environments.'),filters: zod.array(zod.object({diff --git a/js/app/packages/service-clients/service-storage/openapi.json b/js/app/packages/service-clients/service-storage/openapi.jsonindex e3e10f62a6..3d329b1e4f 100644--- a/js/app/packages/service-clients/service-storage/openapi.json+++ b/js/app/packages/service-clients/service-storage/openapi.json@@ -12627,7 +12627,7 @@"properties": {"endpoint_url": {"type": "string",- "description": "HTTPS endpoint URL."+ "description": "Endpoint URL. HTTPS is required outside local environments."},"filters": {"$ref": "#/components/schemas/Vec",@@ -16341,7 +16341,7 @@"properties": {"endpoint_url": {"type": ["string", "null"],- "description": "HTTPS endpoint URL."+ "description": "Endpoint URL. HTTPS is required outside local environments."},"filters": {"oneOf": [@@ -19830,7 +19830,7 @@},"endpoint_url": {"type": "string",- "description": "HTTPS endpoint URL."+ "description": "Endpoint URL. HTTPS is required outside local environments."},"filters": {"$ref": "#/components/schemas/Vec",diff --git a/local_stack.just b/local_stack.justindex effc4afbb7..d80fa3612e 100644--- a/local_stack.just+++ b/local_stack.just@@ -29,6 +29,7 @@ create_local_queues:aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name notification-queue || trueaws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name notification-ingress-queue || trueaws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name push-delivery-queue || true+ aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name webhook-event-queue.fifo --attributes FifoQueue=true || trueaws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name email-service-backfill-queue || trueaws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name delete-chat-handler-queue || trueaws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name contacts-queue || truediff --git a/nix/cloud-storage.nix b/nix/cloud-storage.nixindex 34c694bdef..cc2c9964d6 100644--- a/nix/cloud-storage.nix+++ b/nix/cloud-storage.nix@@ -668,6 +668,7 @@paralleldocker-composecurl+ kcatxzunzipzipdiff --git a/rust/cloud-storage/.sqlx/query-101a8c8116e33aba8ce97f193b8008c5fc34bcf0a874b39ba89f0558771f78d2.json b/rust/cloud-storage/.sqlx/query-101a8c8116e33aba8ce97f193b8008c5fc34bcf0a874b39ba89f0558771f78d2.jsonnew file mode 100644index 0000000000..d1480861e1--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-101a8c8116e33aba8ce97f193b8008c5fc34bcf0a874b39ba89f0558771f78d2.json@@ -0,0 +1,15 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE webhook_delivery\n SET\n status = $2,\n next_attempt_at = NULL,\n updated_at = now()\n WHERE id = $1\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Text",+ "Text"+ ]+ },+ "nullable": []+ },+ "hash": "101a8c8116e33aba8ce97f193b8008c5fc34bcf0a874b39ba89f0558771f78d2"+}diff --git a/rust/cloud-storage/.sqlx/query-1b000e3e86b627dac7ffb345ddc56b1507959639505feedee009c2fa3cea8fdf.json b/rust/cloud-storage/.sqlx/query-1b000e3e86b627dac7ffb345ddc56b1507959639505feedee009c2fa3cea8fdf.jsonnew file mode 100644index 0000000000..8738a45ea4--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-1b000e3e86b627dac7ffb345ddc56b1507959639505feedee009c2fa3cea8fdf.json@@ -0,0 +1,15 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE webhook\n SET\n last_success_at = GREATEST(COALESCE(last_success_at, $2), $2),\n updated_at = GREATEST(updated_at, $2)\n WHERE id = $1\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Text",+ "Timestamptz"+ ]+ },+ "nullable": []+ },+ "hash": "1b000e3e86b627dac7ffb345ddc56b1507959639505feedee009c2fa3cea8fdf"+}diff --git a/rust/cloud-storage/.sqlx/query-35cbfa36e3f14269f1bb5e18ab3a1d67a27f633f50736f8e92c59344b209b801.json b/rust/cloud-storage/.sqlx/query-35cbfa36e3f14269f1bb5e18ab3a1d67a27f633f50736f8e92c59344b209b801.jsonnew file mode 100644index 0000000000..77f31f2e3f--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-35cbfa36e3f14269f1bb5e18ab3a1d67a27f633f50736f8e92c59344b209b801.json@@ -0,0 +1,119 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT\n d.id AS delivery_id,\n d.status AS delivery_status,\n d.attempt_count,\n d.next_attempt_at,\n w.id AS webhook_id,\n w.workspace_id,\n w.name,\n w.endpoint_url,\n w.signing_secret,\n w.headers,\n w.status AS webhook_status,\n w.is_valid,\n w.created_by_user_id,\n w.created_at,\n w.updated_at,\n w.deleted_at,\n w.filters\n FROM webhook_delivery d\n JOIN webhook w ON w.id = d.webhook_id\n WHERE d.webhook_id = $1\n AND d.event_id = $2\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "delivery_id",+ "type_info": "Text"+ },+ {+ "ordinal": 1,+ "name": "delivery_status",+ "type_info": "Text"+ },+ {+ "ordinal": 2,+ "name": "attempt_count",+ "type_info": "Int4"+ },+ {+ "ordinal": 3,+ "name": "next_attempt_at",+ "type_info": "Timestamptz"+ },+ {+ "ordinal": 4,+ "name": "webhook_id",+ "type_info": "Text"+ },+ {+ "ordinal": 5,+ "name": "workspace_id",+ "type_info": "Text"+ },+ {+ "ordinal": 6,+ "name": "name",+ "type_info": "Text"+ },+ {+ "ordinal": 7,+ "name": "endpoint_url",+ "type_info": "Text"+ },+ {+ "ordinal": 8,+ "name": "signing_secret",+ "type_info": "Text"+ },+ {+ "ordinal": 9,+ "name": "headers",+ "type_info": "Jsonb"+ },+ {+ "ordinal": 10,+ "name": "webhook_status",+ "type_info": "Text"+ },+ {+ "ordinal": 11,+ "name": "is_valid",+ "type_info": "Bool"+ },+ {+ "ordinal": 12,+ "name": "created_by_user_id",+ "type_info": "Text"+ },+ {+ "ordinal": 13,+ "name": "created_at",+ "type_info": "Timestamptz"+ },+ {+ "ordinal": 14,+ "name": "updated_at",+ "type_info": "Timestamptz"+ },+ {+ "ordinal": 15,+ "name": "deleted_at",+ "type_info": "Timestamptz"+ },+ {+ "ordinal": 16,+ "name": "filters",+ "type_info": "Jsonb"+ }+ ],+ "parameters": {+ "Left": [+ "Text",+ "Text"+ ]+ },+ "nullable": [+ false,+ false,+ false,+ true,+ false,+ false,+ false,+ false,+ false,+ false,+ false,+ false,+ false,+ false,+ false,+ true,+ false+ ]+ },+ "hash": "35cbfa36e3f14269f1bb5e18ab3a1d67a27f633f50736f8e92c59344b209b801"+}diff --git a/rust/cloud-storage/.sqlx/query-360592f66d7d22bee5c37c9c1a614b382288e75c366ffe18bd663560cb250933.json b/rust/cloud-storage/.sqlx/query-360592f66d7d22bee5c37c9c1a614b382288e75c366ffe18bd663560cb250933.jsonnew file mode 100644index 0000000000..b5b96b9413--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-360592f66d7d22bee5c37c9c1a614b382288e75c366ffe18bd663560cb250933.json@@ -0,0 +1,20 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE webhook_delivery\n SET\n status = $2,\n next_attempt_at = $3,\n delivered_at = $4,\n updated_at = $5\n WHERE id = $1\n AND status = $6\n AND attempt_count = $7\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Text",+ "Text",+ "Timestamptz",+ "Timestamptz",+ "Timestamptz",+ "Text",+ "Int4"+ ]+ },+ "nullable": []+ },+ "hash": "360592f66d7d22bee5c37c9c1a614b382288e75c366ffe18bd663560cb250933"+}diff --git a/rust/cloud-storage/.sqlx/query-366018afc77416e3ef92d6e46f9f1f6f44c74c8461c3110f074a09abc2983a72.json b/rust/cloud-storage/.sqlx/query-366018afc77416e3ef92d6e46f9f1f6f44c74c8461c3110f074a09abc2983a72.jsonnew file mode 100644index 0000000000..11a0712f24--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-366018afc77416e3ef92d6e46f9f1f6f44c74c8461c3110f074a09abc2983a72.json@@ -0,0 +1,32 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE webhook_delivery_attempt\n SET\n status = $4,\n completed_at = now(),\n duration_ms = $5,\n response_status = $6,\n response_headers_redacted = $7,\n response_body_preview = $8,\n error_kind = $9,\n error_message = $10\n WHERE id = $1\n AND webhook_delivery_id = $2\n AND attempt_number = $3\n AND status = $11\n RETURNING completed_at AS \"completed_at!\"\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "completed_at!",+ "type_info": "Timestamptz"+ }+ ],+ "parameters": {+ "Left": [+ "Text",+ "Text",+ "Int4",+ "Text",+ "Int4",+ "Int4",+ "Jsonb",+ "Text",+ "Text",+ "Text",+ "Text"+ ]+ },+ "nullable": [+ true+ ]+ },+ "hash": "366018afc77416e3ef92d6e46f9f1f6f44c74c8461c3110f074a09abc2983a72"+}diff --git a/rust/cloud-storage/.sqlx/query-373397112fdf355b6fa294d50eee28cae14ac674f49c11349823e4b3c2e13938.json b/rust/cloud-storage/.sqlx/query-373397112fdf355b6fa294d50eee28cae14ac674f49c11349823e4b3c2e13938.jsonnew file mode 100644index 0000000000..df24ceafb1--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-373397112fdf355b6fa294d50eee28cae14ac674f49c11349823e4b3c2e13938.json@@ -0,0 +1,15 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE webhook\n SET\n last_failure_at = GREATEST(COALESCE(last_failure_at, $2), $2),\n updated_at = GREATEST(updated_at, $2)\n WHERE id = $1\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Text",+ "Timestamptz"+ ]+ },+ "nullable": []+ },+ "hash": "373397112fdf355b6fa294d50eee28cae14ac674f49c11349823e4b3c2e13938"+}diff --git a/rust/cloud-storage/.sqlx/query-383d8a8ec9c041983d350fda064aefb8d058d7dc39b585bbc9c52070b4eed94f.json b/rust/cloud-storage/.sqlx/query-383d8a8ec9c041983d350fda064aefb8d058d7dc39b585bbc9c52070b4eed94f.jsonnew file mode 100644index 0000000000..f076cc9654--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-383d8a8ec9c041983d350fda064aefb8d058d7dc39b585bbc9c52070b4eed94f.json@@ -0,0 +1,29 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE webhook_delivery\n SET\n status = $2,\n attempt_count = attempt_count + 1,\n first_attempt_at = COALESCE(first_attempt_at, now()),\n last_attempt_at = now(),\n next_attempt_at = NULL,\n updated_at = now()\n WHERE id = $1\n RETURNING\n attempt_count,\n last_attempt_at AS \"started_at!\"\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "attempt_count",+ "type_info": "Int4"+ },+ {+ "ordinal": 1,+ "name": "started_at!",+ "type_info": "Timestamptz"+ }+ ],+ "parameters": {+ "Left": [+ "Text",+ "Text"+ ]+ },+ "nullable": [+ false,+ true+ ]+ },+ "hash": "383d8a8ec9c041983d350fda064aefb8d058d7dc39b585bbc9c52070b4eed94f"+}diff --git a/rust/cloud-storage/.sqlx/query-5dd893b03be604e8ac4d82f90e8f205ea8a52b85cedfb30a314c1b8f891d25ad.json b/rust/cloud-storage/.sqlx/query-5dd893b03be604e8ac4d82f90e8f205ea8a52b85cedfb30a314c1b8f891d25ad.jsonnew file mode 100644index 0000000000..8ac25d6415--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-5dd893b03be604e8ac4d82f90e8f205ea8a52b85cedfb30a314c1b8f891d25ad.json@@ -0,0 +1,40 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT\n webhook_id,\n status,\n attempt_count,\n COALESCE(next_attempt_at <= now(), true) AS \"is_due!\"\n FROM webhook_delivery\n WHERE id = $1\n FOR UPDATE\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "webhook_id",+ "type_info": "Text"+ },+ {+ "ordinal": 1,+ "name": "status",+ "type_info": "Text"+ },+ {+ "ordinal": 2,+ "name": "attempt_count",+ "type_info": "Int4"+ },+ {+ "ordinal": 3,+ "name": "is_due!",+ "type_info": "Bool"+ }+ ],+ "parameters": {+ "Left": [+ "Text"+ ]+ },+ "nullable": [+ false,+ false,+ false,+ null+ ]+ },+ "hash": "5dd893b03be604e8ac4d82f90e8f205ea8a52b85cedfb30a314c1b8f891d25ad"+}diff --git a/rust/cloud-storage/.sqlx/query-5e76ca3bdaae6433ae6091b20624df7c65298da23845b70ed936814b82591dfc.json b/rust/cloud-storage/.sqlx/query-5e76ca3bdaae6433ae6091b20624df7c65298da23845b70ed936814b82591dfc.jsonnew file mode 100644index 0000000000..6eff6ab251--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-5e76ca3bdaae6433ae6091b20624df7c65298da23845b70ed936814b82591dfc.json@@ -0,0 +1,34 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT webhook_id, status, attempt_count\n FROM webhook_delivery\n WHERE id = $1\n FOR UPDATE\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "webhook_id",+ "type_info": "Text"+ },+ {+ "ordinal": 1,+ "name": "status",+ "type_info": "Text"+ },+ {+ "ordinal": 2,+ "name": "attempt_count",+ "type_info": "Int4"+ }+ ],+ "parameters": {+ "Left": [+ "Text"+ ]+ },+ "nullable": [+ false,+ false,+ false+ ]+ },+ "hash": "5e76ca3bdaae6433ae6091b20624df7c65298da23845b70ed936814b82591dfc"+}diff --git a/rust/cloud-storage/.sqlx/query-66c1f8f142ef853861e45f6292689cab233fad58563c99c4f5b349f9b02bc508.json b/rust/cloud-storage/.sqlx/query-66c1f8f142ef853861e45f6292689cab233fad58563c99c4f5b349f9b02bc508.jsonnew file mode 100644index 0000000000..156c4c0178--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-66c1f8f142ef853861e45f6292689cab233fad58563c99c4f5b349f9b02bc508.json@@ -0,0 +1,18 @@+{+ "db_name": "PostgreSQL",+ "query": "\n INSERT INTO webhook_delivery_attempt (\n id,\n webhook_delivery_id,\n attempt_number,\n status,\n started_at\n )\n VALUES ($1, $2, $3, $4, $5)\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Text",+ "Text",+ "Int4",+ "Text",+ "Timestamptz"+ ]+ },+ "nullable": []+ },+ "hash": "66c1f8f142ef853861e45f6292689cab233fad58563c99c4f5b349f9b02bc508"+}diff --git a/rust/cloud-storage/.sqlx/query-683604acb3fb29571b0f79a5bba4fa82f3e2ba9358d051a4249738032f49f06f.json b/rust/cloud-storage/.sqlx/query-683604acb3fb29571b0f79a5bba4fa82f3e2ba9358d051a4249738032f49f06f.jsonnew file mode 100644index 0000000000..65c70b1bea--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-683604acb3fb29571b0f79a5bba4fa82f3e2ba9358d051a4249738032f49f06f.json@@ -0,0 +1,25 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE webhook_delivery_attempt\n SET\n status = $3,\n completed_at = now()\n WHERE webhook_delivery_id = $1\n AND attempt_number = $2\n AND status = $4\n RETURNING completed_at AS \"completed_at!\"\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "completed_at!",+ "type_info": "Timestamptz"+ }+ ],+ "parameters": {+ "Left": [+ "Text",+ "Int4",+ "Text",+ "Text"+ ]+ },+ "nullable": [+ true+ ]+ },+ "hash": "683604acb3fb29571b0f79a5bba4fa82f3e2ba9358d051a4249738032f49f06f"+}diff --git a/rust/cloud-storage/.sqlx/query-7e0d5cd72591beaac66052550a260f7ac3f98e909a1ed7eeea9d321cc600e528.json b/rust/cloud-storage/.sqlx/query-7e0d5cd72591beaac66052550a260f7ac3f98e909a1ed7eeea9d321cc600e528.jsonnew file mode 100644index 0000000000..c95a9be0e4--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-7e0d5cd72591beaac66052550a260f7ac3f98e909a1ed7eeea9d321cc600e528.json@@ -0,0 +1,34 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT webhook_id, status, attempt_count\n FROM webhook_delivery\n WHERE id = $1\n FOR UPDATE\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "webhook_id",+ "type_info": "Text"+ },+ {+ "ordinal": 1,+ "name": "status",+ "type_info": "Text"+ },+ {+ "ordinal": 2,+ "name": "attempt_count",+ "type_info": "Int4"+ }+ ],+ "parameters": {+ "Left": [+ "Text"+ ]+ },+ "nullable": [+ false,+ false,+ false+ ]+ },+ "hash": "7e0d5cd72591beaac66052550a260f7ac3f98e909a1ed7eeea9d321cc600e528"+}diff --git a/rust/cloud-storage/.sqlx/query-9fa8e173d6b50e682450e57c1b5021738e91c289eb7cb1cf26c6637ac9d2ef60.json b/rust/cloud-storage/.sqlx/query-9fa8e173d6b50e682450e57c1b5021738e91c289eb7cb1cf26c6637ac9d2ef60.jsonnew file mode 100644index 0000000000..b4320af041--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-9fa8e173d6b50e682450e57c1b5021738e91c289eb7cb1cf26c6637ac9d2ef60.json@@ -0,0 +1,25 @@+{+ "db_name": "PostgreSQL",+ "query": "\n INSERT INTO webhook_delivery (\n id,\n webhook_id,\n event_id,\n event,\n event_schema_version,\n event_occurred_at,\n event_entity_type,\n event_entity_id,\n event_ordering_key,\n status,\n request_url,\n request_headers_redacted,\n request_body\n )\n SELECT\n $1,\n w.id,\n $3,\n $4,\n $5,\n $6,\n $7,\n $8,\n $9,\n $10,\n w.endpoint_url,\n COALESCE(\n (\n SELECT jsonb_object_agg(header_name, $11::TEXT)\n FROM jsonb_object_keys(w.headers) AS header_name\n ),\n '{}'::JSONB\n ),\n $12\n FROM webhook w\n WHERE w.id = $2\n ON CONFLICT (webhook_id, event_id) DO NOTHING\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Text",+ "Text",+ "Text",+ "Text",+ "Int4",+ "Timestamptz",+ "Text",+ "Text",+ "Text",+ "Text",+ "Text",+ "Jsonb"+ ]+ },+ "nullable": []+ },+ "hash": "9fa8e173d6b50e682450e57c1b5021738e91c289eb7cb1cf26c6637ac9d2ef60"+}diff --git a/rust/cloud-storage/.sqlx/query-f3219580edf757829eb259ed116238fa1f567fab31febb2c7f7400dfcbdb6608.json b/rust/cloud-storage/.sqlx/query-f3219580edf757829eb259ed116238fa1f567fab31febb2c7f7400dfcbdb6608.jsonnew file mode 100644index 0000000000..8acf4324fa--- /dev/null+++ b/rust/cloud-storage/.sqlx/query-f3219580edf757829eb259ed116238fa1f567fab31febb2c7f7400dfcbdb6608.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT DISTINCT team_id::text AS \"team_id!\"\n FROM team_user\n WHERE user_id = ANY($1)\n ORDER BY team_id::text\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "team_id!",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": [+ "TextArray"+ ]+ },+ "nullable": [+ null+ ]+ },+ "hash": "f3219580edf757829eb259ed116238fa1f567fab31febb2c7f7400dfcbdb6608"+}diff --git a/rust/cloud-storage/Cargo.lock b/rust/cloud-storage/Cargo.lockindex 160afff90c..54b7e8d819 100644--- a/rust/cloud-storage/Cargo.lock+++ b/rust/cloud-storage/Cargo.lock@@ -14870,6 +14870,7 @@ dependencies = ["chrono","documents","entity_access",+ "futures","hex","hmac 0.12.1","macro_db_migrator",@@ -14888,6 +14889,7 @@ dependencies = ["serde_json","sha2 0.10.9","sqlx",+ "sqs_client","thiserror 2.0.18","tokio","tower 0.5.3",diff --git a/rust/cloud-storage/document_storage_service/Cargo.toml b/rust/cloud-storage/document_storage_service/Cargo.tomlindex 3afb1df745..1c15f12328 100644--- a/rust/cloud-storage/document_storage_service/Cargo.toml+++ b/rust/cloud-storage/document_storage_service/Cargo.toml@@ -163,7 +163,7 @@ utoipa = { workspace = true, features = ["axum_extras"] }utoipa-swagger-ui = { workspace = true, features = ["axum"] }uuid = { workspace = true }zip = { workspace = true }-webhook = { path = "../webhook", features = ["inbound", "outbound"] }+webhook = { path = "../webhook", features = ["inbound", "outbound", "worker"] }workspace-hack = { version = "0.1", path = "../workspace-hack" }macro_config = { path = "../macro_config" }doppler-config = { workspace = true }diff --git a/rust/cloud-storage/document_storage_service/src/config.rs b/rust/cloud-storage/document_storage_service/src/config.rsindex d878e39c7c..a13f03b471 100644--- a/rust/cloud-storage/document_storage_service/src/config.rs+++ b/rust/cloud-storage/document_storage_service/src/config.rs@@ -125,6 +125,13 @@ pub struct Config {#[macro_config_default(4)]pub queue_wait_time_seconds: i32,+ /// Maximum number of SQS messages to receive per webhook worker poll+ #[macro_config_default(10)]+ pub webhook_queue_max_messages: i32,+ /// SQS long-poll wait time in seconds for the webhook worker+ #[macro_config_default(20)]+ pub webhook_queue_wait_time_seconds: i32,+/// The document limit for free users#[macro_config_default(20)]pub document_limit: u64,diff --git a/rust/cloud-storage/document_storage_service/src/main.rs b/rust/cloud-storage/document_storage_service/src/main.rsindex 4fd027c02e..0105cfd899 100644--- a/rust/cloud-storage/document_storage_service/src/main.rs+++ b/rust/cloud-storage/document_storage_service/src/main.rs@@ -188,6 +188,12 @@ async fn main() -> anyhow::Result<()> {let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(&aws_config)).search_event_queue(&search_event_queue).document_delete_queue(&document_delete_queue);+ let webhook_event_queue = webhook::outbound::SqsWebhookQueue::new(+ Arc::new(sqs_client.clone()),+ macro_queues::WebhookEventQueue::new().to_string(),+ config.webhook_queue_max_messages,+ config.webhook_queue_wait_time_seconds,+ );let contacts_ingress = Arc::new(contacts::domain::service::SqsContactsIngress {queue: contacts::outbound::ingress::SqsContactsQueue::new(@@ -575,11 +581,24 @@ async fn main() -> anyhow::Result<()> {let call_state = CallRouterState::new(call_service.clone(), entity_access_service.clone());let call_webhook_state = WebhookRouterState::new(call_service.clone());- let webhook_service = webhook::domain::service::WebhookServiceImpl::new(- webhook::outbound::pg_repository::PgRepository::new(db.clone()),- webhook::outbound::http_validator::ReqwestWebhookValidationClient::new()- .context("failed to create webhook validation client")?,- );++ let webhook_repository = webhook::outbound::PgRepository::new(db.clone());+ let webhook_endpoint_scheme_policy = if matches!(env, Environment::Local) {+ webhook::domain::models::WebhookEndpointSchemePolicy::HttpAndHttps+ } else {+ webhook::domain::models::WebhookEndpointSchemePolicy::HttpsOnly+ };+ let webhook_http_client =+ webhook::outbound::ReqwestWebhookDeliveryClient::new_with_endpoint_scheme_policy(+ webhook_endpoint_scheme_policy,+ )+ .context("failed to create webhook HTTP client")?;+ let webhook_service =+ webhook::domain::service::WebhookServiceImpl::new_with_endpoint_scheme_policy(+ webhook_repository.clone(),+ webhook_http_client.clone(),+ webhook_endpoint_scheme_policy,+ );let webhook_rate_limiter = RateLimitServiceImpl {repo: RedisRateLimitAdapter {redis: redis_client.clone(),@@ -590,23 +609,50 @@ async fn main() -> anyhow::Result<()> {webhook_rate_limiter,);- // Webhook event ingestion: consume document and channel broker events and- // fan them out to matching webhooks (per-event handlers are stubs for now).let webhook_ingestion_service =webhook::domain::ingestion::WebhookEventIngestionServiceImpl::new(entity_access_service.clone(),+ webhook_repository,+ webhook_event_queue.clone(),);+ let webhook_delivery_repository =+ webhook::outbound::PgWebhookDeliveryRepository::new(db.clone());+ let webhook_delivery_service = webhook::domain::delivery::WebhookEventDeliveryServiceImpl::new(+ webhook_delivery_repository,+ webhook_http_client,+ );+ let webhook_worker = webhook::inbound::worker::WebhookEventWorker::new(+ webhook_event_queue,+ webhook_delivery_service,+ );+ let webhook_worker_task = tokio::spawn(async move {+ tracing::info!("starting webhook event worker");+ webhook_worker.run().await;+ });+ tokio::spawn(async move {+ match webhook_worker_task.await {+ Ok(()) => tracing::error!("webhook event worker exited unexpectedly"),+ Err(error) => {+ tracing::error!(error = ?error, "webhook event worker exited unexpectedly");+ }+ }+ });+let webhook_consumer_brokers = config.kafka_brokers.as_ref().to_string();tokio::spawn(async move {loop {- if let Err(e) = webhook::inbound::kafka_consumer::run_webhook_event_consumer(+ tracing::info!("starting webhook event consumer");+ let result = webhook::inbound::kafka_consumer::run_webhook_event_consumer(&webhook_consumer_brokers,webhook_ingestion_service.clone(),std::future::pending::<()>(),)- .await- {- tracing::error!(error = ?e, "webhook event consumer exited");+ .await;+ match result {+ Ok(()) => tracing::error!("webhook event consumer exited unexpectedly"),+ Err(error) => {+ tracing::error!(error = ?error, "webhook event consumer exited unexpectedly");+ }}tokio::time::sleep(std::time::Duration::from_secs(5)).await;}diff --git a/rust/cloud-storage/macro_queues/src/lib.rs b/rust/cloud-storage/macro_queues/src/lib.rsindex b44a7cc96b..3abe0334b4 100644--- a/rust/cloud-storage/macro_queues/src/lib.rs+++ b/rust/cloud-storage/macro_queues/src/lib.rs@@ -540,6 +540,12 @@ queue! {dev: "push-delivery-queue-dev",prod: "push-delivery-queue-prod",},+ /// FIFO queue for webhook event delivery.+ pub WebhookEventQueue {+ local: "webhook-event-queue.fifo",+ dev: "webhook-event-queue-dev.fifo",+ prod: "webhook-event-queue-prod.fifo",+ },/// Queue for static-file-service S3 upload event notifications.////// Note: this value is consumed as a full SQS URL in deployeddiff --git a/rust/cloud-storage/macro_queues/src/test.rs b/rust/cloud-storage/macro_queues/src/test.rsindex 1b04bcebd7..fbd88eadb8 100644--- a/rust/cloud-storage/macro_queues/src/test.rs+++ b/rust/cloud-storage/macro_queues/src/test.rs@@ -129,6 +129,26 @@ fn grouped_defaults_do_not_check_overrides() {);}+#[test]+fn webhook_event_queue_defaults_are_fifo() {+ assert_eq!(+ WebhookEventQueue::local().as_ref(),+ "webhook-event-queue.fifo"+ );+ assert_eq!(+ WebhookEventQueue::dev().as_ref(),+ "webhook-event-queue-dev.fifo",+ );+ assert_eq!(+ WebhookEventQueue::prod().as_ref(),+ "webhook-event-queue-prod.fifo",+ );+ assert_eq!(+ WebhookEventQueue::local().override_env_var_name(),+ "OVERRIDE_WEBHOOK_EVENT_QUEUE",+ );+}+#[test]fn exported_queues_match_local_values() {let queues = Queues::default_for_environment(Environment::Local);@@ -175,6 +195,10 @@ fn exported_queues_match_local_values() {queues.push_notification_event_handler_queue.as_ref(),"push-delivery-queue",);+ assert_eq!(+ queues.webhook_event_queue.as_ref(),+ "webhook-event-queue.fifo",+ );assert_eq!(queues.static_file_service_s3_event_queue_url.as_ref(),"static-file-s3-event-notification-queue",@@ -254,6 +278,10 @@ fn exported_queues_match_dev_values() {queues.push_notification_event_handler_queue.as_ref(),"push-delivery-queue-dev",);+ assert_eq!(+ queues.webhook_event_queue.as_ref(),+ "webhook-event-queue-dev.fifo",+ );assert_eq!(queues.static_file_service_s3_event_queue_url.as_ref(),"static-file-s3-event-notification-queue-dev",@@ -342,6 +370,10 @@ fn exported_queues_match_prod_values() {queues.push_notification_event_handler_queue.as_ref(),"push-delivery-queue-prod",);+ assert_eq!(+ queues.webhook_event_queue.as_ref(),+ "webhook-event-queue-prod.fifo",+ );assert_eq!(queues.static_file_service_s3_event_queue_url.as_ref(),"static-file-s3-event-notification-queue-prod",@@ -434,6 +466,10 @@ fn exported_queue_override_names_are_derived_from_env_var_names() {PushNotificationEventHandlerQueue::local().override_env_var_name(),"OVERRIDE_PUSH_NOTIFICATION_EVENT_HANDLER_QUEUE",);+ assert_eq!(+ WebhookEventQueue::local().override_env_var_name(),+ "OVERRIDE_WEBHOOK_EVENT_QUEUE",+ );// Pins the digit-boundary case: `S3` stays fused with a separator after it.assert_eq!(StaticFileServiceS3EventQueueUrl::local().override_env_var_name(),diff --git a/rust/cloud-storage/sqs_client/src/lib.rs b/rust/cloud-storage/sqs_client/src/lib.rsindex 8b05979497..3ca831b2bb 100644--- a/rust/cloud-storage/sqs_client/src/lib.rs+++ b/rust/cloud-storage/sqs_client/src/lib.rs@@ -1,4 +1,7 @@mod message_attribute;+mod queue;++pub use queue::ReceivedMessage;#[cfg(feature = "ai_projection")]pub mod ai_projection;diff --git a/rust/cloud-storage/sqs_client/src/queue.rs b/rust/cloud-storage/sqs_client/src/queue.rsnew file mode 100644index 0000000000..80fbbe3699--- /dev/null+++ b/rust/cloud-storage/sqs_client/src/queue.rs@@ -0,0 +1,98 @@+//! Generic SQS transport operations shared by queue-specific adapters.++use crate::SQS;++/// Raw message returned from an SQS receive operation.+#[derive(Clone, Debug, PartialEq, Eq)]+pub struct ReceivedMessage {+ /// Queue-assigned message identifier, when present.+ pub message_id: Option<String>,+ /// Unparsed message body, when present.+ pub body: Option<String>,+ /// Receipt handle used to delete or delay the message, when present.+ pub receipt_handle: Option<String>,+}++impl SQS {+ /// Send one message to a FIFO queue.+ pub async fn send_fifo_message(+ &self,+ queue_url: &str,+ message_body: String,+ message_group_id: String,+ message_deduplication_id: String,+ ) -> anyhow::Result<()> {+ self.inner+ .send_message()+ .queue_url(queue_url)+ .message_body(message_body)+ .message_group_id(message_group_id)+ .message_deduplication_id(message_deduplication_id)+ .send()+ .await?;++ Ok(())+ }++ /// Receive a batch of unparsed messages from a queue.+ pub async fn receive_messages(+ &self,+ queue_url: &str,+ max_number_of_messages: i32,+ wait_time_seconds: i32,+ ) -> anyhow::Result<Vec<ReceivedMessage>> {+ let output = self+ .inner+ .receive_message()+ .queue_url(queue_url)+ .max_number_of_messages(max_number_of_messages)+ .wait_time_seconds(wait_time_seconds)+ .send()+ .await?;++ Ok(output+ .messages+ .unwrap_or_default()+ .into_iter()+ .map(|message| ReceivedMessage {+ message_id: message.message_id,+ body: message.body,+ receipt_handle: message.receipt_handle,+ })+ .collect())+ }++ /// Delete a message from a queue using its receipt handle.+ pub async fn delete_message(+ &self,+ queue_url: &str,+ receipt_handle: &str,+ ) -> anyhow::Result<()> {+ self.inner+ .delete_message()+ .queue_url(queue_url)+ .receipt_handle(receipt_handle)+ .send()+ .await?;++ Ok(())+ }++ /// Change the visibility timeout for a received message.+ pub async fn change_message_visibility(+ &self,+ queue_url: &str,+ receipt_handle: &str,+ visibility_timeout: i32,+ ) -> anyhow::Result<()> {+ self.inner+ .change_message_visibility()+ .queue_url(queue_url)+ .receipt_handle(receipt_handle)+ .visibility_timeout(visibility_timeout)+ .send()+ .await?;++ Ok(())+ }+}diff --git a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/local_env/test.rs b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/local_env/test.rsindex 16482f4bab..98d254afa6 100644--- a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/local_env/test.rs+++ b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/local_env/test.rs@@ -53,6 +53,16 @@ fn values_are_local_only() {}}+#[test]+fn emits_webhook_fifo_queue_override_url() {+ let env = local_env();+ assert_eq!(+ env.get(macro_queues::WebhookEventQueue::OVERRIDE_ENV_VAR_NAME)+ .map(String::as_str),+ Some("http://localstack:4566/000000000000/webhook-event-queue.fifo")+ );+}+#[test]fn aws_creds_are_dummy() {let env = local_env();diff --git a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/localstack.rs b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/localstack.rsindex 54cce20c2c..f3ba39b54a 100644--- a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/localstack.rs+++ b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/localstack.rs@@ -49,10 +49,11 @@ async fn provision_async(url: &str) -> Result<()> {async fn create_queues(sqs: &aws_sdk_sqs::Client) -> Result<()> {for queue in resources::QUEUES {let name = queue.name;- ignore_exists(- sqs.create_queue().queue_name(name).send().await.map(|_| ()),- &format!("queue {name}"),- )?;+ let mut request = sqs.create_queue().queue_name(name);+ if name.ends_with(".fifo") {+ request = request.attributes(aws_sdk_sqs::types::QueueAttributeName::FifoQueue, "true");+ }+ ignore_exists(request.send().await.map(|_| ()), &format!("queue {name}"))?;}Ok(())}diff --git a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/resources.rs b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/resources.rsindex 9f96de1175..28886a2aa2 100644--- a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/resources.rs+++ b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/resources.rs@@ -106,6 +106,10 @@ pub const QUEUES: &[Queue] = &[name: macro_queues::PushNotificationEventHandlerQueue::LOCAL,bindings: &[("PUSH_NOTIFICATION_EVENT_HANDLER_QUEUE", Name)],},+ Queue {+ name: macro_queues::WebhookEventQueue::LOCAL,+ bindings: &[(macro_queues::WebhookEventQueue::OVERRIDE_ENV_VAR_NAME, Url)],+ },Queue {name: macro_queues::EmailBackfillQueue::LOCAL,bindings: &[("BACKFILL_QUEUE", Name), ("EMAIL_BACKFILL_QUEUE", Url)],diff --git a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/resources/test.rs b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/resources/test.rsindex 38493fedf8..b58711add5 100644--- a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/resources/test.rs+++ b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/resources/test.rs@@ -47,3 +47,23 @@ fn every_queue_is_bound() {);}}++#[test]+fn webhook_fifo_queue_uses_a_full_url_override() {+ let queue = QUEUES+ .iter()+ .find(|queue| queue.name == macro_queues::WebhookEventQueue::LOCAL)+ .expect("webhook event queue is cataloged");++ assert!(queue.name.ends_with(".fifo"));+ assert_eq!(queue.bindings.len(), 1);+ assert_eq!(+ queue.bindings[0].0,+ macro_queues::WebhookEventQueue::OVERRIDE_ENV_VAR_NAME+ );+ assert!(matches!(queue.bindings[0].1, QueueForm::Url));+ assert_eq!(+ queue.bindings[0].1.value(queue.name),+ "http://localstack:4566/000000000000/webhook-event-queue.fifo"+ );+}diff --git a/rust/cloud-storage/webhook/Cargo.toml b/rust/cloud-storage/webhook/Cargo.tomlindex f7e2ca3f72..0765ffb539 100644--- a/rust/cloud-storage/webhook/Cargo.toml+++ b/rust/cloud-storage/webhook/Cargo.toml@@ -29,6 +29,7 @@ ingestion = ["dep:channels","dep:documents","dep:entity_access",+ "dep:futures","dep:macro_event_broker","dep:uuid","ports",@@ -40,11 +41,13 @@ outbound = ["dep:sha2","dep:macro_uuid","dep:sqlx",+ "dep:sqs_client","dep:tokio","dep:url","ports",]ports = ["dep:serde_json", "dep:url"]+worker = ["dep:tokio", "ports"][dependencies]anyhow = { workspace = true }@@ -56,6 +59,7 @@ documents = { path = "../documents", default-features = false, optional = true }entity_access = { path = "../entity_access", default-features = false, features = ["ports",], optional = true }+futures = { workspace = true, optional = true }hex = { workspace = true, optional = true }hmac = { workspace = true, optional = true }macro_env = { path = "../macro_env", optional = true }@@ -72,6 +76,7 @@ serde = { workspace = true }serde_json = { workspace = true, optional = true }sha2 = { workspace = true, optional = true }sqlx = { workspace = true, optional = true }+sqs_client = { path = "../sqs_client", default-features = false, optional = true }thiserror = { workspace = true }tokio = { workspace = true, features = ["macros", "time"], optional = true }tracing = { workspace = true }diff --git a/rust/cloud-storage/webhook/README.md b/rust/cloud-storage/webhook/README.mdindex 394c74abd5..91d88f2830 100644--- a/rust/cloud-storage/webhook/README.md+++ b/rust/cloud-storage/webhook/README.md@@ -1,58 +1,247 @@# Webhook crate-This crate currently contains the core application-layer webhook pieces for the first iteration.+This crate provides webhook management, Kafka event ingestion, workspace and+filter matching, FIFO SQS fan-out, tracked HTTP delivery, and retry handling.+`document_storage_service` composes the production adapters and runs both the+Kafka consumer and the queue worker.-## Current scope+## Management and validation-- Database tables are present for event ingestion, webhooks, deliveries, and delivery attempts.-- The Postgres repository supports creating, loading, patching, and updating webhook validity through the `is_valid` field. It also lists active webhooks whose typed `filters` match an event.-- The inbound Axum API supports:- - `POST /webhooks`- - `PATCH /webhooks/{webhook_id}`- - `POST /webhooks/{webhook_id}/validate`-- Validation sends a signed `webhook.validation.test` event to the webhook endpoint and persists the result by updating `is_valid`.-- Validation attempts are rate limited with the existing `rate_limit` crate. The validation key is `per-user-validate-webhook:{macro_user_id}:{webhook_id}` and the current limit is 10 attempts per 3600-second window.+The inbound Axum API supports:-## Typed filters+- `POST /webhooks`+- `PATCH /webhooks/{webhook_id}`+- `DELETE /webhooks/{webhook_id}`+- `POST /webhooks/{webhook_id}/validate`-Webhooks use a typed `filters` array to decide which events are eligible for delivery. Each filter contains event names and may also restrict delivery to specific entity ids:+Validation sends a signed `webhook.validation.test` event to the configured+endpoint and persists the result in `is_valid`. Validation attempts use the+existing `rate_limit` crate with the key+`per-user-validate-webhook:{macro_user_id}:{webhook_id}` and a limit of 10+attempts per 3600-second window.++## Event ingestion and matching++The `webhook-event-ingestion` Kafka consumer reads `macro.documents` and+`macro.channels`. It supports these event names:++- Documents: `document.created`, `document.updated`, `document.deleted`, and+ `document.copied`.+- Channels: `channel.created`, `channel.updated`, `channel.deleted`,+ `channel.message_posted`, `channel.message_patched`,+ `channel.message_deleted`, `channel.message_attachment_created`,+ `channel.message_attachment_removed`, `channel.participant_added`, and+ `channel.participant_removed`.++For each event, ingestion asks `EntityAccessService` for the people who+currently have access to its entity. The matching workspace set contains each+person's Macro user ID, for personal webhooks, plus every team ID to which any+of those people belongs. The set is deduplicated before matching, so one person+or team is considered only once.++A webhook matches only when it is active, valid, not soft-deleted, owned by one+of those personal or team workspaces, and has one filter element that matches+both the exact event name and entity ID. An event match in one filter element+cannot be combined with an ID match in another element.++### Filter-ID semantics++Each filter requires `events`. `ids` is optional; an absent or `null` value+matches every entity ID for that filter's events:```json[{"events": ["document.created"],- "ids": ["doc_123"]+ "ids": ["0197f776-6e7b-7c69-a251-780ae754d3e4"]}]```-`events` is required on every filter. `ids` is optional; when it is absent, that filter matches all entity ids for its events.+For document events, IDs always mean the event's `document_id`. For every+channel event, IDs mean `channel_id`. This includes message, attachment, and+participant events: their message, attachment, and participant IDs are not used+for webhook filtering.++The matching query stores filters in the `webhook.filters` `JSONB NOT NULL`+column, protected by the `webhook_filters_is_array` constraint. It uses the+`webhook_filters_gin_idx` `jsonb_path_ops` GIN index for the event containment+probe and an exact JSONPath recheck to enforce same-filter event/ID matching.++Ingestion concurrently enqueues one message for every matched webhook and waits+for every send. It succeeds only if all sends complete. Repository,+workspace-resolution, queue, and database/internal access failures are+transient: the consumer makes up to five in-process attempts with 1, 2, 4, and+8-second delays. If all five fail, it exits without committing the Kafka offset;+the service supervisor restarts it and Kafka redelivers the event. Invalid+entity IDs, invalid broker contracts, empty Kafka payloads, and undecodable+Kafka messages are traced and skipped so they do not block a partition.++## Queue contract, deduplication, and ordering++The versioned SQS message contains a webhook ID and a normalized event. The+event retains the broker event ID, schema version, event name, entity type and+ID, entity ordering key, ingestion timestamp, and complete broker envelope.+The normalized `occurred_at` value, persisted as `event_occurred_at`, is the+time of webhook ingestion rather than an event-producer timestamp. Endpoint+URLs, custom headers, and signing secrets are deliberately omitted; the worker+reloads the webhook's current configuration before delivery.++The webhook event queue is FIFO. Each send explicitly uses:-## Database storage and matching+- `MessageGroupId`: the webhook ID.+- `MessageDeduplicationId`: `<webhook_id>:<event_id>`.-Filters are stored in the `webhook.filters` `JSONB NOT NULL` column. The column has the `webhook_filters_is_array` check constraint so stored filter data is always an array:+This gives each webhook its own FIFO group and suppresses duplicate enqueue+operations during SQS's deduplication interval. The database additionally has a+unique `(webhook_id, event_id)` delivery constraint, so duplicate queue receipts+reuse the same delivery record.-```sql-CHECK (jsonb_typeof(filters) = 'array')+FIFO preserves the order in which the current consumer enqueues events for one+webhook. It does not create a global order across Kafka partitions or across the+`macro.documents` and `macro.channels` topics. Consumers must therefore treat+the order as observed order, not total event order.++## HTTP delivery contract++Delivery is an HTTPS `POST` with a five-second timeout and redirects disabled.+The complete broker envelope is serialized as the exact request body; normalized+queue and delivery fields are not wrapped around or merged into it. Its shape is:++```json+{+ "event_id": "0197f776-6e7b-7c69-a251-780ae754d3e4",+ "schema_version": 1,+ "event_type": "document.deleted",+ "metadata": {+ "document_id": "0197f776-6e7b-7c69-a251-780ae754d3e4",+ "actor_user_id": null,+ "project_id": null+ }+}```-The `webhook_filters_gin_idx` index is a GIN index using `jsonb_path_ops` on `filters`. It supports containment probes on keys inside filter elements, including `events` and `ids`. The canonical matching query used by `WebhookRepo::list_active_webhooks_matching_event` combines an index-backed containment predicate for the event with an exact JSONPath recheck for the same filter element's event and entity-id semantics:+The request includes:++- `Content-Type: application/json`+- `X-Macro-Event`: the exact event name+- `X-Macro-Event-Id`: the broker event ID+- `X-Macro-Timestamp`: the request time as Unix seconds+- `X-Macro-Signature`: `v1=<hex HMAC-SHA256>`++The signature input is the raw byte sequence+`<X-Macro-Timestamp>.<request-body>`, using the webhook signing secret. Custom+headers are included, but case-insensitive attempts to override a Macro header+are permanent configuration failures. Delivery applies the same HTTPS, DNS,+blocked-address, reserved-header, signing, and no-redirect protections as+validation.++Every 2xx response succeeds. Network errors, timeouts, DNS resolution failures,+and HTTP 408, 429, and 5xx responses are retryable. Other non-2xx responses and+invalid endpoint or header configuration are permanent failures. Delivery+attempts store response status, redacted response-header values, sanitized error+details, duration, and at most 4096 bytes of a UTF-8-lossy response-body preview.++## Delivery state, retries, and cancellation++The worker prepares a delivery idempotently, reloads the webhook, and records+each HTTP attempt in Postgres. Terminal delivered, canceled, permanently failed,+and exhausted deliveries are acknowledged without another HTTP request.+Successful attempts update `webhook.last_success_at`; attempted failures update+`webhook.last_failure_at`.++A delivery receives at most five HTTP attempts. Retryable failures use this+schedule:++| Failed attempt | Delay before next attempt |+| --- | --- |+| 1 | 30 seconds |+| 2 | 60 seconds |+| 3 | 120 seconds |+| 4 | 300 seconds |+| 5 | No retry; mark exhausted |++A permanent failure is recorded and acknowledged immediately. If the webhook is+missing, the message is acknowledged. If its current configuration is paused,+disabled, invalid, or soft-deleted, the delivery is marked canceled where+possible and acknowledged without consuming an HTTP attempt.++Delivery failures update tracking data but do not automatically pause or+disable a webhook. Automatic pause/disable policy is intentionally deferred;+operators or webhook owners must currently change status explicitly.++## Worker and at-least-once behavior++The worker long-polls in batches and processes each received batch sequentially.+A missing body, malformed JSON body, or unsupported queue contract version is a+poison message: it is traced and deleted immediately when SQS supplied a receipt+handle. A service, delete, or visibility-update failure is traced and leaves the+message for SQS redelivery. In deployed environments, repeatedly unacknowledged+messages move to the FIFO dead-letter queue after 20 receives.++The pipeline is at-least-once. SQS deduplication and the database uniqueness+constraint suppress ordinary duplicates, and a terminal database record+prevents another request. There is still an unavoidable crash window: an+endpoint can accept the HTTP request before the process records the result. A+redelivery can then recover the interrupted attempt and call the endpoint again.+Webhook receivers should use `X-Macro-Event-Id` as an idempotency key.++## Operations++### Queue configuration and local setup++`macro_queues::WebhookEventQueue` selects these FIFO queue names:++| Environment | Queue name |+| --- | --- |+| Local | `webhook-event-queue.fifo` |+| Develop | `webhook-event-queue-dev.fifo` |+| Production | `webhook-event-queue-prod.fifo` |++`OVERRIDE_WEBHOOK_EVENT_QUEUE` takes precedence over the environment default.+The webhook crate's `outbound::SqsWebhookQueue` wraps the shared+`sqs_client::SQS` and owns webhook-specific FIFO serialization, polling,+acknowledgment, and visibility changes.+Set the override to the full queue URL when connecting to LocalStack. From the+repository root, the canonical local orchestrator provisions the FIFO queue and+exports this value automatically:++```sh+just run_local --no-frontend+```-```sql-WHERE w.filters @> jsonb_build_array(jsonb_build_object('events', jsonb_build_array($2::text)))- AND jsonb_path_exists(- w.filters,- '$[*] ? (@.events[*] == $event && (!exists (@.ids) || @.ids == null || @.ids[*] == $id))',- jsonb_build_object('event', $2::text, 'id', $3::text)- )+Services inside the local compose network receive+`http://localstack:4566/000000000000/webhook-event-queue.fifo`. The+`docker-compose.local-e2e.yml` environment supplies the same override. Worker+polling defaults to 10 messages and a 20-second long poll and can be adjusted+with `WEBHOOK_QUEUE_MAX_MESSAGES` and `WEBHOOK_QUEUE_WAIT_TIME_SECONDS`.++### Tracing++Use a targeted `RUST_LOG` filter while diagnosing ingestion or delivery, for+example:++```text+RUST_LOG=info,webhook=trace,webhook::outbound::sqs_queue=trace```-## Temporary limitations+Ingestion spans expose event ID/name, entity type/ID, accessor count, workspace+count, match count, and webhook ID. Worker and delivery spans expose queue+message ID, webhook/event/delivery/attempt IDs, attempt number, delivery status,+HTTP status, duration, and retry delay. Queue polling and acknowledgment errors+are also traced. Request and response bodies, signing secrets, signatures, and+custom header values must not be added to logs; the implemented spans skip+those values.-- Webhook IDs currently use prefixed UUIDv7 strings (`wh_<uuid_v7>`). They can be swapped for true ULIDs later.-- Webhook signing secrets are generated by the service and stored in plaintext in `signing_secret`, matching the existing bot secret storage approach.-- Custom headers are stored as JSON in the `headers` column.+For durable investigation, correlate `webhook_delivery` and+`webhook_delivery_attempt` rows by webhook/event ID and inspect the FIFO DLQ for+transport failures that survived 20 receives.-## Intentionally out of scope+## Temporary limitations-Delivery workers, SQS/SNS infrastructure, event ingestion, redelivery, and auto-pause behavior are intentionally out of scope for this crate iteration.+- Webhook, delivery, and attempt IDs use prefixed UUIDv7 strings rather than true+ ULIDs.+- Webhook signing secrets are generated by the service and stored in plaintext+ in `signing_secret`, matching the existing bot secret storage approach.+- Custom headers are stored as JSON in the `headers` column; persisted delivery+ snapshots and response headers redact their values.diff --git a/rust/cloud-storage/webhook/src/domain/delivery.rs b/rust/cloud-storage/webhook/src/domain/delivery.rsnew file mode 100644index 0000000000..ceba8f9265--- /dev/null+++ b/rust/cloud-storage/webhook/src/domain/delivery.rs@@ -0,0 +1,288 @@+//! Webhook event delivery state machine and retry policy.++#[cfg(test)]+mod test;++use super::{+ models::{+ PreparedWebhookDelivery, WebhookDeliveryAttempt, WebhookDeliveryStatus,+ WebhookEventQueueMessage, WebhookHttpOutcome, WebhookHttpOutcomeDetails, WebhookStatus,+ WebhookWorkerDisposition,+ },+ ports::{WebhookDeliveryClient, WebhookDeliveryRepository, WebhookEventDeliveryService},+};+use chrono::Utc;+use std::time::Duration;++const MAX_HTTP_ATTEMPTS: u32 = 5;+const RETRY_DELAYS: [Duration; 4] = [+ Duration::from_secs(30),+ Duration::from_secs(60),+ Duration::from_secs(120),+ Duration::from_secs(300),+];++/// Error returned while processing a queued webhook event.+#[derive(Debug, thiserror::Error)]+pub enum WebhookEventDeliveryError {+ /// The delivery repository failed to prepare or persist delivery state.+ #[error("webhook delivery repository failed: {0}")]+ Repository(#[source] anyhow::Error),+ /// The HTTP client encountered an internal failure with no delivery outcome.+ #[error("webhook delivery client failed: {0}")]+ Client(#[source] anyhow::Error),+ /// A non-terminal delivery has already reached the domain's attempt limit.+ #[error("webhook delivery has already started {0} attempts; maximum is five")]+ AttemptLimitReached(u32),+ /// The repository returned an attempt outside the domain's one-based limit.+ #[error("webhook delivery repository returned invalid attempt number {0}")]+ InvalidAttemptNumber(u32),+ /// The repository declined a due, non-terminal delivery without exposing updated state.+ #[error("webhook delivery repository did not begin a due delivery attempt")]+ AttemptNotStarted,+}++/// Domain implementation of webhook event delivery and retry policy.+#[derive(Debug, Clone)]+pub struct WebhookEventDeliveryServiceImpl<R, C> {+ repository: R,+ client: C,+}++impl<R, C> WebhookEventDeliveryServiceImpl<R, C> {+ /// Create a webhook event delivery service.+ pub fn new(repository: R, client: C) -> Self {+ Self { repository, client }+ }+}++impl<R, C> WebhookEventDeliveryServiceImpl<R, C>+where+ R: WebhookDeliveryRepository,+ C: WebhookDeliveryClient,+{+ fn repository_error(error: R::Err) -> WebhookEventDeliveryError {+ WebhookEventDeliveryError::Repository(error.into())+ }++ fn client_error(error: C::Err) -> WebhookEventDeliveryError {+ WebhookEventDeliveryError::Client(error.into())+ }++ async fn cancel_ineligible_delivery(+ &self,+ delivery_id: &str,+ ) -> Result<WebhookWorkerDisposition, WebhookEventDeliveryError> {+ self.repository+ .cancel_delivery(delivery_id)+ .await+ .map_err(Self::repository_error)?;+ record_delivery_status(WebhookDeliveryStatus::Canceled.as_str());+ Ok(WebhookWorkerDisposition::Acknowledge)+ }++ async fn disposition_after_declined_attempt(+ &self,+ message: &WebhookEventQueueMessage,+ ) -> Result<WebhookWorkerDisposition, WebhookEventDeliveryError> {+ let Some(prepared) = self+ .repository+ .prepare_delivery(message)+ .await+ .map_err(Self::repository_error)?+ else {+ record_delivery_status("missing");+ return Ok(WebhookWorkerDisposition::Acknowledge);+ };++ tracing::Span::current().record("delivery_id", prepared.delivery_id.as_str());+ record_delivery_status(prepared.status.as_str());++ if prepared.status.is_terminal() {+ return Ok(WebhookWorkerDisposition::Acknowledge);+ }+ if !webhook_is_eligible(&prepared) {+ return self.cancel_ineligible_delivery(&prepared.delivery_id).await;+ }+ if let Some(delay) = remaining_delay(&prepared) {+ record_retry_delay(delay);+ return Ok(WebhookWorkerDisposition::RetryAfter(delay));+ }++ Err(WebhookEventDeliveryError::AttemptNotStarted)+ }++ async fn record_outcome(+ &self,+ attempt: &WebhookDeliveryAttempt,+ outcome: WebhookHttpOutcome,+ ) -> Result<WebhookWorkerDisposition, WebhookEventDeliveryError> {+ match outcome {+ WebhookHttpOutcome::Success(details) => {+ record_http_details(&details);+ self.repository+ .record_success(attempt, &details)+ .await+ .map_err(Self::repository_error)?;+ record_delivery_status(WebhookDeliveryStatus::Delivered.as_str());+ Ok(WebhookWorkerDisposition::Acknowledge)+ }+ WebhookHttpOutcome::PermanentFailure(details) => {+ record_http_details(&details);+ self.repository+ .record_permanent_failure(attempt, &details)+ .await+ .map_err(Self::repository_error)?;+ record_delivery_status(WebhookDeliveryStatus::PermanentlyFailed.as_str());+ Ok(WebhookWorkerDisposition::Acknowledge)+ }+ WebhookHttpOutcome::RetryableFailure(details)+ if attempt.attempt_number == MAX_HTTP_ATTEMPTS =>+ {+ record_http_details(&details);+ self.repository+ .record_exhaustion(attempt, &details)+ .await+ .map_err(Self::repository_error)?;+ record_delivery_status(WebhookDeliveryStatus::Exhausted.as_str());+ Ok(WebhookWorkerDisposition::Acknowledge)+ }+ WebhookHttpOutcome::RetryableFailure(details) => {+ record_http_details(&details);+ let delay = retry_delay(attempt.attempt_number).ok_or(+ WebhookEventDeliveryError::InvalidAttemptNumber(attempt.attempt_number),+ )?;+ let next_attempt_at = Utc::now()+ + chrono::Duration::from_std(delay)+ .expect("webhook retry delays fit in chrono::Duration");++ self.repository+ .record_retryable_failure(attempt, &details, next_attempt_at)+ .await+ .map_err(Self::repository_error)?;+ record_delivery_status(WebhookDeliveryStatus::RetryScheduled.as_str());+ record_retry_delay(delay);+ Ok(WebhookWorkerDisposition::RetryAfter(delay))+ }+ }+ }+}++impl<R, C> WebhookEventDeliveryService for WebhookEventDeliveryServiceImpl<R, C>+where+ R: WebhookDeliveryRepository,+ C: WebhookDeliveryClient,+{+ type Err = WebhookEventDeliveryError;++ #[tracing::instrument(+ skip(self, message),+ fields(+ webhook_id = %message.webhook_id,+ event_id = %message.event.event_id,+ delivery_id = tracing::field::Empty,+ attempt_id = tracing::field::Empty,+ attempt_number = tracing::field::Empty,+ status = tracing::field::Empty,+ http_status = tracing::field::Empty,+ duration_ms = tracing::field::Empty,+ retry_delay_seconds = tracing::field::Empty,+ ),+ err+ )]+ async fn deliver_event(+ &self,+ message: WebhookEventQueueMessage,+ ) -> Result<WebhookWorkerDisposition, Self::Err> {+ let Some(prepared) = self+ .repository+ .prepare_delivery(&message)+ .await+ .map_err(Self::repository_error)?+ else {+ record_delivery_status("missing");+ return Ok(WebhookWorkerDisposition::Acknowledge);+ };++ tracing::Span::current().record("delivery_id", prepared.delivery_id.as_str());+ record_delivery_status(prepared.status.as_str());++ if prepared.status.is_terminal() {+ return Ok(WebhookWorkerDisposition::Acknowledge);+ }+ if !webhook_is_eligible(&prepared) {+ return self.cancel_ineligible_delivery(&prepared.delivery_id).await;+ }+ if let Some(delay) = remaining_delay(&prepared) {+ record_retry_delay(delay);+ return Ok(WebhookWorkerDisposition::RetryAfter(delay));+ }+ if prepared.attempt_count >= MAX_HTTP_ATTEMPTS {+ return Err(WebhookEventDeliveryError::AttemptLimitReached(+ prepared.attempt_count,+ ));+ }++ let Some(attempt) = self+ .repository+ .begin_attempt(&prepared.delivery_id)+ .await+ .map_err(Self::repository_error)?+ else {+ return self.disposition_after_declined_attempt(&message).await;+ };++ if !(1..=MAX_HTTP_ATTEMPTS).contains(&attempt.attempt_number) {+ return Err(WebhookEventDeliveryError::InvalidAttemptNumber(+ attempt.attempt_number,+ ));+ }++ tracing::Span::current().record("attempt_id", attempt.attempt_id.as_str());+ tracing::Span::current().record("attempt_number", attempt.attempt_number);+ record_delivery_status(WebhookDeliveryStatus::InProgress.as_str());++ let outcome = self+ .client+ .deliver(&prepared.webhook, &message.event)+ .await+ .map_err(Self::client_error)?;++ self.record_outcome(&attempt, outcome).await+ }+}++fn webhook_is_eligible(prepared: &PreparedWebhookDelivery) -> bool {+ prepared.webhook.status == WebhookStatus::Active+ && prepared.webhook.is_valid+ && prepared.webhook.deleted_at.is_none()+}++fn retry_delay(attempt_number: u32) -> Option<Duration> {+ let delay_index = usize::try_from(attempt_number.checked_sub(1)?).ok()?;+ RETRY_DELAYS.get(delay_index).copied()+}++fn remaining_delay(prepared: &PreparedWebhookDelivery) -> Option<Duration> {+ let next_attempt_at = prepared.next_attempt_at?;+ next_attempt_at+ .signed_duration_since(Utc::now())+ .to_std()+ .ok()+}++fn record_delivery_status(status: &str) {+ tracing::Span::current().record("status", status);+}++fn record_http_details(details: &WebhookHttpOutcomeDetails) {+ if let Some(status) = details.response_status {+ tracing::Span::current().record("http_status", status);+ }+ let duration_ms = u64::try_from(details.duration.as_millis()).unwrap_or(u64::MAX);+ tracing::Span::current().record("duration_ms", duration_ms);+}++fn record_retry_delay(delay: Duration) {+ tracing::Span::current().record("retry_delay_seconds", delay.as_secs());+}diff --git a/rust/cloud-storage/webhook/src/domain/delivery/test.rs b/rust/cloud-storage/webhook/src/domain/delivery/test.rsnew file mode 100644index 0000000000..da3149b60f--- /dev/null+++ b/rust/cloud-storage/webhook/src/domain/delivery/test.rs@@ -0,0 +1,616 @@+use super::*;+use crate::domain::{+ models::{+ NormalizedWebhookEvent, Webhook, WebhookDeliveryAttempt, WebhookDeliveryStatus,+ WebhookEventQueueMessage, WebhookHeaders, WebhookHttpOutcome, WebhookHttpOutcomeDetails,+ WebhookStatus, WebhookWorkerDisposition,+ },+ ports::{WebhookDeliveryClient, WebhookDeliveryRepository, WebhookEventDeliveryService},+};+use chrono::{DateTime, Utc};+use serde_json::json;+use std::{+ collections::{BTreeMap, VecDeque},+ sync::{Arc, Mutex, MutexGuard},+ time::Duration,+};++const WEBHOOK_ID: &str = "wh_delivery_service";+const DELIVERY_ID: &str = "whd_delivery_service";+const EVENT_ID: &str = "evt_delivery_service";++fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {+ mutex.lock().expect("test mutex is not poisoned")+}++fn timestamp() -> DateTime<Utc> {+ DateTime::parse_from_rfc3339("2026-01-02T03:04:05Z")+ .expect("valid timestamp")+ .with_timezone(&Utc)+}++fn message() -> WebhookEventQueueMessage {+ WebhookEventQueueMessage::new(+ WEBHOOK_ID.to_string(),+ NormalizedWebhookEvent {+ event_id: EVENT_ID.to_string(),+ schema_version: 1,+ event_name: "document.created".to_string(),+ entity_type: "document".to_string(),+ entity_id: "doc_123".to_string(),+ ordering_key: "doc_123".to_string(),+ occurred_at: timestamp(),+ broker_envelope: json!({"event_id": EVENT_ID, "metadata": {"id": "doc_123"}}),+ },+ )+}++fn webhook(endpoint_url: &str) -> Webhook {+ Webhook {+ id: WEBHOOK_ID.to_string(),+ workspace_id: "macro|[email protected]".to_string(),+ name: "Delivery test".to_string(),+ endpoint_url: endpoint_url.to_string(),+ signing_secret: "whsec_current".to_string(),+ headers: BTreeMap::new(),+ status: WebhookStatus::Active,+ is_valid: true,+ created_by_user_id: "macro|[email protected]".to_string(),+ created_at: timestamp(),+ updated_at: timestamp(),+ deleted_at: None,+ filters: Vec::new(),+ }+}++fn prepared(status: WebhookDeliveryStatus, attempt_count: u32) -> PreparedWebhookDelivery {+ PreparedWebhookDelivery {+ delivery_id: DELIVERY_ID.to_string(),+ webhook: webhook("https://current.example.com/webhook"),+ status,+ attempt_count,+ next_attempt_at: None,+ }+}++fn attempt(attempt_number: u32) -> WebhookDeliveryAttempt {+ WebhookDeliveryAttempt {+ attempt_id: format!("wha_{attempt_number}"),+ delivery_id: DELIVERY_ID.to_string(),+ attempt_number,+ }+}++fn details(response_status: Option<u16>) -> WebhookHttpOutcomeDetails {+ WebhookHttpOutcomeDetails {+ duration: Duration::from_millis(125),+ response_status,+ response_headers_redacted: Some(WebhookHeaders::from([(+ "content-type".to_string(),+ "[REDACTED]".to_string(),+ )])),+ response_body_preview: None,+ error_kind: None,+ error_message: None,+ }+}++#[derive(Default)]+struct MockRepositoryState {+ prepared: Option<PreparedWebhookDelivery>,+ prepared_responses: VecDeque<Option<PreparedWebhookDelivery>>,+ begin_responses: VecDeque<Option<WebhookDeliveryAttempt>>,+ prepare_calls: Vec<WebhookEventQueueMessage>,+ begin_calls: Vec<String>,+ canceled_delivery_ids: Vec<String>,+ successes: Vec<(WebhookDeliveryAttempt, WebhookHttpOutcomeDetails)>,+ retries: Vec<(+ WebhookDeliveryAttempt,+ WebhookHttpOutcomeDetails,+ DateTime<Utc>,+ )>,+ permanent_failures: Vec<(WebhookDeliveryAttempt, WebhookHttpOutcomeDetails)>,+ exhaustions: Vec<(WebhookDeliveryAttempt, WebhookHttpOutcomeDetails)>,+ fail_prepare: bool,+ fail_begin: bool,+ fail_record: bool,+}++#[derive(Clone, Default)]+struct MockRepository {+ state: Arc<Mutex<MockRepositoryState>>,+}++impl MockRepository {+ fn for_attempt(prepared: PreparedWebhookDelivery, attempt_number: u32) -> Self {+ let repository = Self::default();+ {+ let mut state = lock(&repository.state);+ state.prepared = Some(prepared);+ state+ .begin_responses+ .push_back(Some(attempt(attempt_number)));+ }+ repository+ }+}++impl WebhookDeliveryRepository for MockRepository {+ type Err = anyhow::Error;++ async fn prepare_delivery(+ &self,+ message: &WebhookEventQueueMessage,+ ) -> Result<Option<PreparedWebhookDelivery>, Self::Err> {+ let mut state = lock(&self.state);+ state.prepare_calls.push(message.clone());+ if state.fail_prepare {+ anyhow::bail!("prepare failed");+ }+ if let Some(response) = state.prepared_responses.pop_front() {+ return Ok(response);+ }+ Ok(state.prepared.clone())+ }++ async fn cancel_delivery(&self, delivery_id: &str) -> Result<(), Self::Err> {+ lock(&self.state)+ .canceled_delivery_ids+ .push(delivery_id.to_string());+ Ok(())+ }++ async fn begin_attempt(+ &self,+ delivery_id: &str,+ ) -> Result<Option<WebhookDeliveryAttempt>, Self::Err> {+ let mut state = lock(&self.state);+ state.begin_calls.push(delivery_id.to_string());+ if state.fail_begin {+ anyhow::bail!("begin failed");+ }+ Ok(state.begin_responses.pop_front().flatten())+ }++ async fn record_success(+ &self,+ attempt: &WebhookDeliveryAttempt,+ details: &WebhookHttpOutcomeDetails,+ ) -> Result<(), Self::Err> {+ let mut state = lock(&self.state);+ if state.fail_record {+ anyhow::bail!("record failed");+ }+ state.successes.push((attempt.clone(), details.clone()));+ Ok(())+ }++ async fn record_retryable_failure(+ &self,+ attempt: &WebhookDeliveryAttempt,+ details: &WebhookHttpOutcomeDetails,+ next_attempt_at: DateTime<Utc>,+ ) -> Result<(), Self::Err> {+ let mut state = lock(&self.state);+ if state.fail_record {+ anyhow::bail!("record failed");+ }+ state+ .retries+ .push((attempt.clone(), details.clone(), next_attempt_at));+ Ok(())+ }++ async fn record_permanent_failure(+ &self,+ attempt: &WebhookDeliveryAttempt,+ details: &WebhookHttpOutcomeDetails,+ ) -> Result<(), Self::Err> {+ let mut state = lock(&self.state);+ if state.fail_record {+ anyhow::bail!("record failed");+ }+ state+ .permanent_failures+ .push((attempt.clone(), details.clone()));+ Ok(())+ }++ async fn record_exhaustion(+ &self,+ attempt: &WebhookDeliveryAttempt,+ details: &WebhookHttpOutcomeDetails,+ ) -> Result<(), Self::Err> {+ let mut state = lock(&self.state);+ if state.fail_record {+ anyhow::bail!("record failed");+ }+ state.exhaustions.push((attempt.clone(), details.clone()));+ Ok(())+ }+}++enum MockClientResponse {+ Outcome(WebhookHttpOutcome),+ Error,+}++#[derive(Default)]+struct MockClientState {+ responses: VecDeque<MockClientResponse>,+ requests: Vec<(Webhook, NormalizedWebhookEvent)>,+}++#[derive(Clone, Default)]+struct MockClient {+ state: Arc<Mutex<MockClientState>>,+}++impl MockClient {+ fn with_outcome(outcome: WebhookHttpOutcome) -> Self {+ let client = Self::default();+ lock(&client.state)+ .responses+ .push_back(MockClientResponse::Outcome(outcome));+ client+ }+}++impl WebhookDeliveryClient for MockClient {+ type Err = anyhow::Error;++ async fn deliver(+ &self,+ webhook: &Webhook,+ event: &NormalizedWebhookEvent,+ ) -> Result<WebhookHttpOutcome, Self::Err> {+ let mut state = lock(&self.state);+ state.requests.push((webhook.clone(), event.clone()));+ match state.responses.pop_front() {+ Some(MockClientResponse::Outcome(outcome)) => Ok(outcome),+ Some(MockClientResponse::Error) => anyhow::bail!("internal client failure"),+ None => anyhow::bail!("no mock client response"),+ }+ }+}++#[tokio::test]+async fn successful_delivery_is_recorded_and_acknowledged() {+ let repository = MockRepository::for_attempt(prepared(WebhookDeliveryStatus::Queued, 0), 1);+ let success_details = details(Some(204));+ let client = MockClient::with_outcome(WebhookHttpOutcome::Success(success_details.clone()));+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client.clone());++ let disposition = service.deliver_event(message()).await.expect("delivery");++ assert_eq!(disposition, WebhookWorkerDisposition::Acknowledge);+ let repository_state = lock(&repository.state);+ assert_eq!(+ repository_state.successes,+ vec![(attempt(1), success_details)]+ );+ assert!(repository_state.retries.is_empty());+ assert_eq!(lock(&client.state).requests.len(), 1);+}++#[tokio::test]+async fn permanent_failure_is_recorded_and_acknowledged() {+ let repository = MockRepository::for_attempt(prepared(WebhookDeliveryStatus::Queued, 0), 1);+ let failure_details = details(Some(400));+ let client = MockClient::with_outcome(WebhookHttpOutcome::PermanentFailure(+ failure_details.clone(),+ ));+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client);++ let disposition = service.deliver_event(message()).await.expect("delivery");++ assert_eq!(disposition, WebhookWorkerDisposition::Acknowledge);+ assert_eq!(+ lock(&repository.state).permanent_failures,+ vec![(attempt(1), failure_details)]+ );+}++#[tokio::test]+async fn retryable_failures_use_the_exact_delay_schedule() {+ for (attempt_number, expected_seconds) in [(1, 30), (2, 60), (3, 120), (4, 300)] {+ let repository = MockRepository::for_attempt(+ prepared(+ if attempt_number == 1 {+ WebhookDeliveryStatus::Queued+ } else {+ WebhookDeliveryStatus::RetryScheduled+ },+ attempt_number - 1,+ ),+ attempt_number,+ );+ let failure_details = details(Some(503));+ let client = MockClient::with_outcome(WebhookHttpOutcome::RetryableFailure(+ failure_details.clone(),+ ));+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client);+ let expected_delay = Duration::from_secs(expected_seconds);+ let earliest_due =+ Utc::now() + chrono::Duration::from_std(expected_delay).expect("valid test delay");++ let disposition = service.deliver_event(message()).await.expect("delivery");++ let latest_due =+ Utc::now() + chrono::Duration::from_std(expected_delay).expect("valid test delay");+ assert_eq!(+ disposition,+ WebhookWorkerDisposition::RetryAfter(expected_delay)+ );+ let repository_state = lock(&repository.state);+ assert_eq!(repository_state.retries.len(), 1);+ let (recorded_attempt, recorded_details, next_attempt_at) = &repository_state.retries[0];+ assert_eq!(recorded_attempt, &attempt(attempt_number));+ assert_eq!(recorded_details, &failure_details);+ assert!(*next_attempt_at >= earliest_due);+ assert!(*next_attempt_at <= latest_due);+ assert!(repository_state.exhaustions.is_empty());+ }+}++#[tokio::test]+async fn fifth_retryable_failure_is_exhausted_and_acknowledged() {+ let repository =+ MockRepository::for_attempt(prepared(WebhookDeliveryStatus::RetryScheduled, 4), 5);+ let failure_details = details(Some(429));+ let client = MockClient::with_outcome(WebhookHttpOutcome::RetryableFailure(+ failure_details.clone(),+ ));+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client);++ let disposition = service.deliver_event(message()).await.expect("delivery");++ assert_eq!(disposition, WebhookWorkerDisposition::Acknowledge);+ let repository_state = lock(&repository.state);+ assert_eq!(+ repository_state.exhaustions,+ vec![(attempt(5), failure_details)]+ );+ assert!(repository_state.retries.is_empty());+}++#[tokio::test]+async fn terminal_deliveries_suppress_duplicate_http_attempts() {+ for status in [+ WebhookDeliveryStatus::Delivered,+ WebhookDeliveryStatus::Canceled,+ WebhookDeliveryStatus::PermanentlyFailed,+ WebhookDeliveryStatus::Exhausted,+ ] {+ let repository = MockRepository::default();+ lock(&repository.state).prepared = Some(prepared(status, 1));+ let client = MockClient::default();+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client.clone());++ let disposition = service.deliver_event(message()).await.expect("delivery");++ assert_eq!(disposition, WebhookWorkerDisposition::Acknowledge);+ assert!(lock(&repository.state).begin_calls.is_empty());+ assert!(lock(&client.state).requests.is_empty());+ }+}++#[tokio::test]+async fn every_due_non_terminal_status_can_start_an_attempt() {+ for (status, attempt_count, attempt_number) in [+ (WebhookDeliveryStatus::Queued, 0, 1),+ (WebhookDeliveryStatus::InProgress, 1, 2),+ (WebhookDeliveryStatus::RetryScheduled, 2, 3),+ ] {+ let repository =+ MockRepository::for_attempt(prepared(status, attempt_count), attempt_number);+ let client = MockClient::with_outcome(WebhookHttpOutcome::Success(details(Some(200))));+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client);++ let disposition = service.deliver_event(message()).await.expect("delivery");++ assert_eq!(disposition, WebhookWorkerDisposition::Acknowledge);+ assert_eq!(lock(&repository.state).successes.len(), 1);+ }+}++#[tokio::test]+async fn future_retry_returns_its_remaining_delay_without_an_attempt() {+ let repository = MockRepository::default();+ let due_at = Utc::now() + chrono::Duration::hours(1);+ let mut future_delivery = prepared(WebhookDeliveryStatus::RetryScheduled, 1);+ future_delivery.next_attempt_at = Some(due_at);+ lock(&repository.state).prepared = Some(future_delivery);+ let client = MockClient::default();+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client.clone());++ let disposition = service.deliver_event(message()).await.expect("delivery");++ let WebhookWorkerDisposition::RetryAfter(delay) = disposition else {+ panic!("future delivery should be retried later");+ };+ assert!(delay <= Duration::from_secs(3600));+ assert!(delay > Duration::from_secs(3599));+ assert!(lock(&repository.state).begin_calls.is_empty());+ assert!(lock(&client.state).requests.is_empty());+}++#[tokio::test]+async fn ineligible_webhooks_are_canceled_without_http_attempts() {+ let mut ineligible_webhooks = Vec::new();++ let mut paused = webhook("https://paused.example.com");+ paused.status = WebhookStatus::Paused;+ ineligible_webhooks.push(paused);++ let mut disabled = webhook("https://disabled.example.com");+ disabled.status = WebhookStatus::Disabled;+ ineligible_webhooks.push(disabled);++ let mut invalid = webhook("https://invalid.example.com");+ invalid.is_valid = false;+ ineligible_webhooks.push(invalid);++ let mut deleted = webhook("https://deleted.example.com");+ deleted.deleted_at = Some(Utc::now());+ ineligible_webhooks.push(deleted);++ for ineligible_webhook in ineligible_webhooks {+ let repository = MockRepository::default();+ let mut delivery = prepared(WebhookDeliveryStatus::Queued, 0);+ delivery.webhook = ineligible_webhook;+ lock(&repository.state).prepared = Some(delivery);+ let client = MockClient::default();+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client.clone());++ let disposition = service.deliver_event(message()).await.expect("delivery");++ assert_eq!(disposition, WebhookWorkerDisposition::Acknowledge);+ let repository_state = lock(&repository.state);+ assert_eq!(+ repository_state.canceled_delivery_ids,+ vec![DELIVERY_ID.to_string()]+ );+ assert!(repository_state.begin_calls.is_empty());+ assert!(lock(&client.state).requests.is_empty());+ }+}++#[tokio::test]+async fn missing_webhook_is_acknowledged_without_an_http_attempt() {+ let repository = MockRepository::default();+ let client = MockClient::default();+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client.clone());++ let disposition = service.deliver_event(message()).await.expect("delivery");++ assert_eq!(disposition, WebhookWorkerDisposition::Acknowledge);+ let repository_state = lock(&repository.state);+ assert!(repository_state.canceled_delivery_ids.is_empty());+ assert!(repository_state.begin_calls.is_empty());+ assert!(lock(&client.state).requests.is_empty());+}++#[tokio::test]+async fn each_queue_receipt_uses_refreshed_webhook_configuration() {+ let repository = MockRepository::default();+ let mut first = prepared(WebhookDeliveryStatus::Queued, 0);+ first.webhook.endpoint_url = "https://old.example.com/webhook".to_string();+ first.webhook.signing_secret = "whsec_old".to_string();+ let mut refreshed = prepared(WebhookDeliveryStatus::Queued, 0);+ refreshed.webhook.endpoint_url = "https://new.example.com/webhook".to_string();+ refreshed.webhook.signing_secret = "whsec_new".to_string();+ refreshed.webhook.headers = BTreeMap::from([("X-Current".to_string(), "yes".to_string())]);+ {+ let mut state = lock(&repository.state);+ state.prepared_responses.push_back(Some(first));+ state.prepared_responses.push_back(Some(refreshed));+ state.begin_responses.push_back(Some(attempt(1)));+ state.begin_responses.push_back(Some(attempt(1)));+ }++ let client = MockClient::default();+ {+ let mut state = lock(&client.state);+ state+ .responses+ .push_back(MockClientResponse::Outcome(WebhookHttpOutcome::Success(+ details(Some(200)),+ )));+ state+ .responses+ .push_back(MockClientResponse::Outcome(WebhookHttpOutcome::Success(+ details(Some(200)),+ )));+ }+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client.clone());++ service+ .deliver_event(message())+ .await+ .expect("first receipt");+ service+ .deliver_event(message())+ .await+ .expect("second receipt");++ assert_eq!(lock(&repository.state).prepare_calls.len(), 2);+ let client_state = lock(&client.state);+ assert_eq!(client_state.requests.len(), 2);+ assert_eq!(+ client_state.requests[0].0.endpoint_url,+ "https://old.example.com/webhook"+ );+ assert_eq!(client_state.requests[0].0.signing_secret, "whsec_old");+ assert_eq!(+ client_state.requests[1].0.endpoint_url,+ "https://new.example.com/webhook"+ );+ assert_eq!(client_state.requests[1].0.signing_secret, "whsec_new");+ assert_eq!(+ client_state.requests[1].0.headers.get("X-Current"),+ Some(&"yes".to_string())+ );+}++#[tokio::test]+async fn repository_and_internal_client_failures_remain_unacknowledged() {+ let prepare_failure_repository = MockRepository::default();+ lock(&prepare_failure_repository.state).fail_prepare = true;+ let prepare_failure_service =+ WebhookEventDeliveryServiceImpl::new(prepare_failure_repository, MockClient::default());+ assert!(matches!(+ prepare_failure_service.deliver_event(message()).await,+ Err(WebhookEventDeliveryError::Repository(_))+ ));++ let client_failure_repository =+ MockRepository::for_attempt(prepared(WebhookDeliveryStatus::Queued, 0), 1);+ let client = MockClient::default();+ lock(&client.state)+ .responses+ .push_back(MockClientResponse::Error);+ let client_failure_service =+ WebhookEventDeliveryServiceImpl::new(client_failure_repository.clone(), client);+ assert!(matches!(+ client_failure_service.deliver_event(message()).await,+ Err(WebhookEventDeliveryError::Client(_))+ ));+ let repository_state = lock(&client_failure_repository.state);+ assert!(repository_state.successes.is_empty());+ assert!(repository_state.retries.is_empty());+ assert!(repository_state.permanent_failures.is_empty());+ assert!(repository_state.exhaustions.is_empty());+}++#[tokio::test]+async fn non_terminal_delivery_at_the_attempt_limit_does_not_start_a_sixth_attempt() {+ let repository = MockRepository::for_attempt(+ prepared(WebhookDeliveryStatus::InProgress, MAX_HTTP_ATTEMPTS),+ MAX_HTTP_ATTEMPTS + 1,+ );+ let client = MockClient::with_outcome(WebhookHttpOutcome::Success(details(Some(200))));+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client.clone());++ assert!(matches!(+ service.deliver_event(message()).await,+ Err(WebhookEventDeliveryError::AttemptLimitReached(+ MAX_HTTP_ATTEMPTS+ ))+ ));+ assert!(lock(&repository.state).begin_calls.is_empty());+ assert!(lock(&client.state).requests.is_empty());+}++#[tokio::test]+async fn retry_is_returned_only_after_the_retry_state_is_recorded() {+ let repository = MockRepository::for_attempt(prepared(WebhookDeliveryStatus::Queued, 0), 1);+ lock(&repository.state).fail_record = true;+ let client = MockClient::with_outcome(WebhookHttpOutcome::RetryableFailure(details(Some(503))));+ let service = WebhookEventDeliveryServiceImpl::new(repository, client);++ assert!(matches!(+ service.deliver_event(message()).await,+ Err(WebhookEventDeliveryError::Repository(_))+ ));+}diff --git a/rust/cloud-storage/webhook/src/domain/ingestion.rs b/rust/cloud-storage/webhook/src/domain/ingestion.rsindex 3d6834a0e2..b25da39a5d 100644--- a/rust/cloud-storage/webhook/src/domain/ingestion.rs+++ b/rust/cloud-storage/webhook/src/domain/ingestion.rs@@ -1,72 +1,83 @@-//! Webhook event ingestion: broker events in, matching webhooks out.+//! Webhook event ingestion: broker events in, matching queue messages out.//!//! `WebhookEventIngestionService` is the inbound port driven by broker//! consumers (see `crate::inbound::kafka_consumer`). It is intentionally//! separate from `WebhookService`, which is the CRUD surface for webhook//! creation and edits.-//!-//! Event handling is not implemented yet: every handler below is a stub that-//! only logs the event. The planned flow for each event is:-//! 1. resolve the users that can see the event's entity via-//! `EntityAccessService::get_users_by_entity` (see-//! `WebhookEventIngestionServiceImpl::users_with_access`);-//! 2. call `WebhookRepo::list_active_webhooks_matching_event` to load those-//! users' active webhooks whose typed `filters` match the event name and-//! entity id;-//! 3. enqueue a delivery for every match.#[cfg(test)]mod test;-use channels::domain::broker_events::{- ChannelCreatedMetadata, ChannelDeletedMetadata, ChannelMessageAttachmentCreatedMetadata,- ChannelMessageAttachmentRemovedMetadata, ChannelMessageDeletedMetadata,- ChannelMessagePatchedMetadata, ChannelMessagePostedMetadata, ChannelParticipantAddedMetadata,- ChannelParticipantRemovedMetadata, ChannelTopicEvent, ChannelUpdatedMetadata,-};-use documents::domain::events::{- DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata, DocumentTopicEvent,- DocumentUpdatedMetadata,+use crate::domain::{+ models::{NormalizedWebhookEvent, WebhookEventQueueMessage},+ ports::{WebhookEventEnqueuer, WebhookRepo, WebhookWorkspaceResolver},};+use channels::domain::broker_events::ChannelTopicEvent;+use chrono::Utc;+use documents::domain::events::DocumentTopicEvent;use entity_access::domain::models::{AccessError, EntityType};use entity_access::domain::ports::EntityAccessService;+use futures::future::join_all;use macro_event_broker::Event;use macro_user_id::user_id::MacroUserIdStr;use std::future::Future;use std::sync::Arc;+use tracing::Instrument as _;use uuid::Uuid;+const DOCUMENT_ENTITY_TYPE: &str = "document";+const CHANNEL_ENTITY_TYPE: &str = "channel";+/// Webhook event ingestion error.#[derive(Debug, thiserror::Error)]pub enum WebhookEventIngestionError {/// Failed to resolve the users with access to the event's entity.#[error(transparent)]EntityAccess(#[from] AccessError),- /// Repository or adapter error.- #[error(transparent)]- Internal(#[from] anyhow::Error),+ /// The event's entity identifier violates the broker contract.+ #[error("invalid {entity_type} entity id: {entity_id}")]+ InvalidEntityId {+ /// Contract entity type.+ entity_type: &'static str,+ /// Invalid entity identifier.+ entity_id: String,+ },+ /// The broker envelope could not be represented by the queue contract.+ #[error("failed to serialize broker envelope: {0}")]+ Serialization(#[from] serde_json::Error),+ /// Failed to resolve personal and team webhook workspaces.+ #[error("failed to resolve webhook workspaces: {0}")]+ WorkspaceResolution(#[source] anyhow::Error),+ /// Failed to find active webhooks matching the event.+ #[error("failed to match active webhooks: {0}")]+ Repository(#[source] anyhow::Error),+ /// Failed to enqueue at least one matched webhook event.+ #[error("failed to enqueue webhook event: {0}")]+ Enqueue(#[source] anyhow::Error),}impl WebhookEventIngestionError {/// Whether retrying the same event could plausibly succeed.///- /// Database errors from access resolution are transient: the lookup can be- /// retried once the database recovers, and the consumer holds off on- /// committing the event's offset until it does. Everything else — invalid- /// entity ids, unauthorized lookups, adapter bugs — is permanent, and- /// retrying would only repeat the failure.+ /// Access resolution currently maps some PostgreSQL failures to+ /// [`AccessError::Internal`], so that variant remains retryable alongside+ /// explicit database, repository, workspace-resolution, and queue errors.+ /// Invalid broker contracts are permanent and can be safely skipped.pub fn is_transient(&self) -> bool {- match self {- Self::EntityAccess(AccessError::DatabaseError(_)) => true,- Self::EntityAccess(_) | Self::Internal(_) => false,- }+ matches!(+ self,+ Self::EntityAccess(AccessError::DatabaseError(_) | AccessError::Internal)+ | Self::WorkspaceResolution(_)+ | Self::Repository(_)+ | Self::Enqueue(_)+ )}}/// Inbound port for ingesting broker events for webhook delivery.////// One method per subscribed topic; each takes the decoded event envelope so-/// implementations keep the `event_id` for delivery idempotency.+/// implementations retain the `event_id` and complete original payload.pub trait WebhookEventIngestionService: Clone + Send + Sync + 'static {/// Ingest one `macro.documents` event envelope.fn ingest_document_event(@@ -83,254 +94,209 @@ pub trait WebhookEventIngestionService: Clone + Send + Sync + 'static {/// Webhook event ingestion service implementation.#[derive(Clone)]-pub struct WebhookEventIngestionServiceImpl<A> {+pub struct WebhookEventIngestionServiceImpl<A, R, Q> {entity_access_service: Arc<A>,+ repository: R,+ enqueuer: Q,}-impl<A> WebhookEventIngestionServiceImpl<A> {+impl<A, R, Q> WebhookEventIngestionServiceImpl<A, R, Q> {/// Create a webhook event ingestion service.- pub fn new(entity_access_service: Arc<A>) -> Self {+ pub fn new(entity_access_service: Arc<A>, repository: R, enqueuer: Q) -> Self {Self {entity_access_service,+ repository,+ enqueuer,}}}-impl<A: EntityAccessService> WebhookEventIngestionServiceImpl<A> {+impl<A, R, Q> WebhookEventIngestionServiceImpl<A, R, Q>+where+ A: EntityAccessService,+ R: WebhookRepo + WebhookWorkspaceResolver,+ Q: WebhookEventEnqueuer,+{/// Resolve the users that currently have access to an entity.- ///- /// This is the first step of webhook fan-out: the handlers below will use- /// it to find the users whose webhooks should be matched against an event.#[tracing::instrument(skip(self), err)]pub async fn users_with_access(&self,entity_id: &str,entity_type: EntityType,) -> Result<Vec<MacroUserIdStr<'static>>, WebhookEventIngestionError> {- tracing::trace!(entity_id, ?entity_type, "resolving users with access");- let users = self- .entity_access_service+ self.entity_access_service.get_users_by_entity(entity_id, entity_type)- .await?;- tracing::trace!(- entity_id,- user_count = users.len(),- "resolved users with access"- );- Ok(users)- }-- async fn handle_document_created(- &self,- event_id: Uuid,- metadata: DocumentCreatedMetadata,- ) -> Result<(), WebhookEventIngestionError> {- tracing::debug!(- %event_id,- document_id = %metadata.document_id,- "document.created: webhook ingestion not implemented yet"- );- Ok(())+ .await+ .map_err(WebhookEventIngestionError::EntityAccess)}- async fn handle_document_updated(+ #[tracing::instrument(+ skip_all,+ fields(+ event_id = %event.event_id,+ event_name = %event.event_name,+ entity_type = %event.entity_type,+ entity_id = %event.entity_id,+ accessor_count = tracing::field::Empty,+ workspace_count = tracing::field::Empty,+ match_count = tracing::field::Empty,+ ),+ err+ )]+ async fn ingest_normalized_event(&self,- event_id: Uuid,- metadata: DocumentUpdatedMetadata,- ) -> Result<(), WebhookEventIngestionError> {- tracing::debug!(- %event_id,- document_id = %metadata.document_id,- "document.updated: webhook ingestion not implemented yet"- );- Ok(())- }-- async fn handle_document_deleted(- &self,- event_id: Uuid,- metadata: DocumentDeletedMetadata,+ event: NormalizedWebhookEvent,+ entity_type: EntityType,) -> Result<(), WebhookEventIngestionError> {- tracing::debug!(- %event_id,- document_id = %metadata.document_id,- "document.deleted: webhook ingestion not implemented yet"- );- Ok(())- }+ let accessors = self+ .users_with_access(&event.entity_id, entity_type)+ .await?;+ tracing::Span::current().record("accessor_count", accessors.len());- async fn handle_document_copied(- &self,- event_id: Uuid,- metadata: DocumentCopiedMetadata,- ) -> Result<(), WebhookEventIngestionError> {- tracing::debug!(- %event_id,- document_id = %metadata.document_id,- source_document_id = %metadata.source_document_id,- "document.copied: webhook ingestion not implemented yet"- );- Ok(())- }+ let workspace_ids = self+ .repository+ .resolve_workspace_ids(accessors)+ .await+ .map_err(|error| WebhookEventIngestionError::WorkspaceResolution(error.into()))?;+ tracing::Span::current().record("workspace_count", workspace_ids.len());- async fn handle_channel_created(- &self,- event_id: Uuid,- metadata: ChannelCreatedMetadata,- ) -> Result<(), WebhookEventIngestionError> {- tracing::debug!(- %event_id,- channel_id = %metadata.channel_id,- "channel.created: webhook ingestion not implemented yet"- );- Ok(())- }+ let webhooks = self+ .repository+ .list_active_webhooks_matching_event(+ workspace_ids,+ event.event_name.clone(),+ event.entity_id.clone(),+ )+ .await+ .map_err(|error| WebhookEventIngestionError::Repository(error.into()))?;+ tracing::Span::current().record("match_count", webhooks.len());- async fn handle_channel_updated(- &self,- event_id: Uuid,- metadata: ChannelUpdatedMetadata,- ) -> Result<(), WebhookEventIngestionError> {- tracing::debug!(- %event_id,- channel_id = %metadata.channel_id,- "channel.updated: webhook ingestion not implemented yet"- );- Ok(())- }+ let enqueue_results = join_all(webhooks.into_iter().map(|webhook| {+ let enqueuer = self.enqueuer.clone();+ let webhook_id = webhook.id;+ let message = WebhookEventQueueMessage::new(webhook_id.clone(), event.clone());+ async move {+ enqueuer+ .enqueue(message)+ .instrument(tracing::info_span!("enqueue_webhook_event", %webhook_id))+ .await+ .map_err(Into::into)+ }+ }))+ .await;- async fn handle_channel_deleted(- &self,- event_id: Uuid,- metadata: ChannelDeletedMetadata,- ) -> Result<(), WebhookEventIngestionError> {- tracing::debug!(- %event_id,- channel_id = %metadata.channel_id,- "channel.deleted: webhook ingestion not implemented yet"- );- Ok(())- }+ for result in enqueue_results {+ result.map_err(WebhookEventIngestionError::Enqueue)?;+ }- async fn handle_channel_message_posted(- &self,- event_id: Uuid,- metadata: ChannelMessagePostedMetadata,- ) -> Result<(), WebhookEventIngestionError> {- tracing::debug!(- %event_id,- channel_id = %metadata.channel_id,- message_id = %metadata.message_id,- "channel.message_posted: webhook ingestion not implemented yet"- );Ok(())}+}- async fn handle_channel_message_patched(- &self,- event_id: Uuid,- metadata: ChannelMessagePatchedMetadata,- ) -> Result<(), WebhookEventIngestionError> {- tracing::debug!(- %event_id,- channel_id = %metadata.channel_id,- message_id = %metadata.message_id,- "channel.message_patched: webhook ingestion not implemented yet"- );- Ok(())- }+fn normalized_document_event(+ event: &Event<DocumentTopicEvent>,+) -> Result<NormalizedWebhookEvent, WebhookEventIngestionError> {+ let (event_name, entity_id) = match &event.event {+ DocumentTopicEvent::Created(metadata) => ("document.created", &metadata.document_id),+ DocumentTopicEvent::Updated(metadata) => ("document.updated", &metadata.document_id),+ DocumentTopicEvent::Deleted(metadata) => ("document.deleted", &metadata.document_id),+ DocumentTopicEvent::Copied(metadata) => ("document.copied", &metadata.document_id),+ };- async fn handle_channel_message_deleted(- &self,- event_id: Uuid,- metadata: ChannelMessageDeletedMetadata,- ) -> Result<(), WebhookEventIngestionError> {- tracing::debug!(- %event_id,- channel_id = %metadata.channel_id,- message_id = %metadata.message_id,- "channel.message_deleted: webhook ingestion not implemented yet"- );- Ok(())+ if Uuid::parse_str(entity_id).is_err() {+ return Err(WebhookEventIngestionError::InvalidEntityId {+ entity_type: DOCUMENT_ENTITY_TYPE,+ entity_id: entity_id.clone(),+ });}- async fn handle_channel_message_attachment_created(- &self,- event_id: Uuid,- metadata: ChannelMessageAttachmentCreatedMetadata,- ) -> Result<(), WebhookEventIngestionError> {- tracing::debug!(- %event_id,- channel_id = %metadata.channel_id,- message_id = %metadata.message_id,- "channel.message_attachment_created: webhook ingestion not implemented yet"- );- Ok(())- }+ let broker_envelope = serde_json::to_value(event)?;+ Ok(normalized_event(+ event.event_id,+ event.schema_version,+ event_name,+ DOCUMENT_ENTITY_TYPE,+ entity_id,+ broker_envelope,+ ))+}- async fn handle_channel_message_attachment_removed(- &self,- event_id: Uuid,- metadata: ChannelMessageAttachmentRemovedMetadata,- ) -> Result<(), WebhookEventIngestionError> {- tracing::debug!(- %event_id,- channel_id = %metadata.channel_id,- message_id = %metadata.message_id,- "channel.message_attachment_removed: webhook ingestion not implemented yet"- );- Ok(())- }+fn normalized_channel_event(+ event: &Event<ChannelTopicEvent>,+) -> Result<NormalizedWebhookEvent, WebhookEventIngestionError> {+ let (event_name, channel_id) = match &event.event {+ ChannelTopicEvent::Created(metadata) => ("channel.created", metadata.channel_id),+ ChannelTopicEvent::Updated(metadata) => ("channel.updated", metadata.channel_id),+ ChannelTopicEvent::Deleted(metadata) => ("channel.deleted", metadata.channel_id),+ ChannelTopicEvent::MessagePosted(metadata) => {+ ("channel.message_posted", metadata.channel_id)+ }+ ChannelTopicEvent::MessagePatched(metadata) => {+ ("channel.message_patched", metadata.channel_id)+ }+ ChannelTopicEvent::MessageDeleted(metadata) => {+ ("channel.message_deleted", metadata.channel_id)+ }+ ChannelTopicEvent::MessageAttachmentCreated(metadata) => {+ ("channel.message_attachment_created", metadata.channel_id)+ }+ ChannelTopicEvent::MessageAttachmentRemoved(metadata) => {+ ("channel.message_attachment_removed", metadata.channel_id)+ }+ ChannelTopicEvent::ParticipantAdded(metadata) => {+ ("channel.participant_added", metadata.channel_id)+ }+ ChannelTopicEvent::ParticipantRemoved(metadata) => {+ ("channel.participant_removed", metadata.channel_id)+ }+ };+ let entity_id = channel_id.to_string();- async fn handle_channel_participant_added(- &self,- event_id: Uuid,- metadata: ChannelParticipantAddedMetadata,- ) -> Result<(), WebhookEventIngestionError> {- tracing::debug!(- %event_id,- channel_id = %metadata.channel_id,- "channel.participant_added: webhook ingestion not implemented yet"- );- Ok(())- }+ let broker_envelope = serde_json::to_value(event)?;+ Ok(normalized_event(+ event.event_id,+ event.schema_version,+ event_name,+ CHANNEL_ENTITY_TYPE,+ &entity_id,+ broker_envelope,+ ))+}- async fn handle_channel_participant_removed(- &self,- event_id: Uuid,- metadata: ChannelParticipantRemovedMetadata,- ) -> Result<(), WebhookEventIngestionError> {- tracing::debug!(- %event_id,- channel_id = %metadata.channel_id,- "channel.participant_removed: webhook ingestion not implemented yet"- );- Ok(())+fn normalized_event(+ event_id: Uuid,+ schema_version: u8,+ event_name: &str,+ entity_type: &str,+ entity_id: &str,+ broker_envelope: serde_json::Value,+) -> NormalizedWebhookEvent {+ NormalizedWebhookEvent {+ event_id: event_id.to_string(),+ schema_version,+ event_name: event_name.to_string(),+ entity_type: entity_type.to_string(),+ entity_id: entity_id.to_string(),+ ordering_key: entity_id.to_string(),+ occurred_at: Utc::now(),+ broker_envelope,}}-impl<A: EntityAccessService> WebhookEventIngestionService for WebhookEventIngestionServiceImpl<A> {+impl<A, R, Q> WebhookEventIngestionService for WebhookEventIngestionServiceImpl<A, R, Q>+where+ A: EntityAccessService,+ R: WebhookRepo + WebhookWorkspaceResolver,+ Q: WebhookEventEnqueuer,+{#[tracing::instrument(skip(self, event), fields(event_id = %event.event_id), err)]async fn ingest_document_event(&self,event: Event<DocumentTopicEvent>,) -> Result<(), WebhookEventIngestionError> {- tracing::trace!(?event, "ingesting document event");- let event_id = event.event_id;- match event.event {- DocumentTopicEvent::Created(metadata) => {- self.handle_document_created(event_id, metadata).await- }- DocumentTopicEvent::Updated(metadata) => {- self.handle_document_updated(event_id, metadata).await- }- DocumentTopicEvent::Deleted(metadata) => {- self.handle_document_deleted(event_id, metadata).await- }- DocumentTopicEvent::Copied(metadata) => {- self.handle_document_copied(event_id, metadata).await- }- }+ let event = normalized_document_event(&event)?;+ self.ingest_normalized_event(event, EntityType::Document)+ .await}#[tracing::instrument(skip(self, event), fields(event_id = %event.event_id), err)]@@ -338,45 +304,8 @@ impl<A: EntityAccessService> WebhookEventIngestionService for WebhookEventIngest&self,event: Event<ChannelTopicEvent>,) -> Result<(), WebhookEventIngestionError> {- tracing::trace!(?event, "ingesting channel event");- let event_id = event.event_id;- match event.event {- ChannelTopicEvent::Created(metadata) => {- self.handle_channel_created(event_id, metadata).await- }- ChannelTopicEvent::Updated(metadata) => {- self.handle_channel_updated(event_id, metadata).await- }- ChannelTopicEvent::Deleted(metadata) => {- self.handle_channel_deleted(event_id, metadata).await- }- ChannelTopicEvent::MessagePosted(metadata) => {- self.handle_channel_message_posted(event_id, metadata).await- }- ChannelTopicEvent::MessagePatched(metadata) => {- self.handle_channel_message_patched(event_id, metadata)- .await- }- ChannelTopicEvent::MessageDeleted(metadata) => {- self.handle_channel_message_deleted(event_id, metadata)- .await- }- ChannelTopicEvent::MessageAttachmentCreated(metadata) => {- self.handle_channel_message_attachment_created(event_id, metadata)- .await- }- ChannelTopicEvent::MessageAttachmentRemoved(metadata) => {- self.handle_channel_message_attachment_removed(event_id, metadata)- .await- }- ChannelTopicEvent::ParticipantAdded(metadata) => {- self.handle_channel_participant_added(event_id, metadata)- .await- }- ChannelTopicEvent::ParticipantRemoved(metadata) => {- self.handle_channel_participant_removed(event_id, metadata)- .await- }- }+ let event = normalized_channel_event(&event)?;+ self.ingest_normalized_event(event, EntityType::Channel)+ .await}}diff --git a/rust/cloud-storage/webhook/src/domain/ingestion/test.rs b/rust/cloud-storage/webhook/src/domain/ingestion/test.rsindex 955bc25612..a75a8b2db4 100644--- a/rust/cloud-storage/webhook/src/domain/ingestion/test.rs+++ b/rust/cloud-storage/webhook/src/domain/ingestion/test.rs@@ -1,12 +1,45 @@use super::*;+use crate::domain::{+ models::{+ CreateWebhookRequest, PatchWebhookRequest, Webhook, WebhookEventQueueMessage, WebhookStatus,+ },+ ports::{WebhookEventEnqueuer, WebhookRepo, WebhookWorkspaceResolver},+};use channel_sender::ChannelSender;-use channels::domain::models::ChannelType;+use channels::domain::{+ broker_events::{+ ChannelCreatedMetadata, ChannelDeletedMetadata, ChannelMessageAttachmentCreatedMetadata,+ ChannelMessageAttachmentRemovedMetadata, ChannelMessageDeletedMetadata,+ ChannelMessagePatchedMetadata, ChannelMessagePostedMetadata,+ ChannelParticipantAddedMetadata, ChannelParticipantRemovedMetadata, ChannelUpdatedMetadata,+ },+ models::ChannelType,+};use chrono::{DateTime, Utc};+use documents::domain::events::{+ DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata,+ DocumentUpdatedMetadata,+};use entity_access::domain::models::{AccessLevel, CallChannelInfo, EntityAccessReceipt, EntityPermission, RequiredPermission,UserTeamInfo,};use macro_user_id::{lowercased::Lowercase, user_id::MacroUserId};+use serde_json::Value;+use std::{+ collections::BTreeMap,+ sync::{Arc, Mutex, MutexGuard},+ time::Duration,+};++const DOCUMENT_ID: &str = "11111111-1111-1111-1111-111111111111";+const COPIED_DOCUMENT_ID: &str = "22222222-2222-2222-2222-222222222222";+const PERSONAL_WORKSPACE_ID: &str = "macro|[email protected]";+const TEAM_WORKSPACE_ID: &str = "33333333-3333-3333-3333-333333333333";++fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {+ mutex.lock().expect("test mutex is not poisoned")+}fn user_id(id: &str) -> MacroUserIdStr<'static> {MacroUserIdStr::try_from(id.to_string()).expect("valid user id")@@ -22,10 +55,27 @@ fn timestamp() -> DateTime<Utc> {.with_timezone(&Utc)}-/// Mock access service: only [`EntityAccessService::get_users_by_entity`] is-/// exercised by the ingestion service.+#[derive(Debug, Clone, Copy)]+enum AccessFailure {+ Internal,+}+#[derive(Clone)]-struct MockAccessService;+struct MockAccessService {+ users: Vec<MacroUserIdStr<'static>>,+ failure: Option<AccessFailure>,+ calls: Arc<Mutex<Vec<(String, EntityType)>>>,+}++impl MockAccessService {+ fn with_users(users: Vec<MacroUserIdStr<'static>>) -> Self {+ Self {+ users,+ failure: None,+ calls: Arc::default(),+ }+ }+}impl EntityAccessService for MockAccessService {async fn generate_entity_access_receipt<T: RequiredPermission>(@@ -87,10 +137,14 @@ impl EntityAccessService for MockAccessService {async fn get_users_by_entity(&self,- _entity_id: &str,- _entity_type: EntityType,+ entity_id: &str,+ entity_type: EntityType,) -> Result<Vec<MacroUserIdStr<'static>>, AccessError> {- Ok(vec![user_id("macro|[email protected]")])+ lock(&self.calls).push((entity_id.to_string(), entity_type));+ match self.failure {+ Some(AccessFailure::Internal) => Err(AccessError::Internal),+ None => Ok(self.users.clone()),+ }}async fn get_call_channel(@@ -115,166 +169,749 @@ impl EntityAccessService for MockAccessService {}}-fn service() -> WebhookEventIngestionServiceImpl<MockAccessService> {- WebhookEventIngestionServiceImpl::new(Arc::new(MockAccessService))+#[derive(Debug, Clone, PartialEq, Eq)]+struct MatchCall {+ workspace_ids: Vec<String>,+ event_name: String,+ entity_id: String,+}++struct MockRepositoryState {+ resolved_workspace_ids: Vec<String>,+ matching_webhooks: Vec<Webhook>,+ fail_workspace_resolution: bool,+ fail_matching: bool,+ workspace_calls: Vec<Vec<MacroUserIdStr<'static>>>,+ match_calls: Vec<MatchCall>,+}++#[derive(Clone)]+struct MockRepository {+ state: Arc<Mutex<MockRepositoryState>>,+}++impl MockRepository {+ fn new(resolved_workspace_ids: Vec<String>, matching_webhooks: Vec<Webhook>) -> Self {+ Self {+ state: Arc::new(Mutex::new(MockRepositoryState {+ resolved_workspace_ids,+ matching_webhooks,+ fail_workspace_resolution: false,+ fail_matching: false,+ workspace_calls: Vec::new(),+ match_calls: Vec::new(),+ })),+ }+ }+}++impl WebhookWorkspaceResolver for MockRepository {+ type Err = anyhow::Error;++ async fn resolve_workspace_ids(+ &self,+ people: Vec<MacroUserIdStr<'static>>,+ ) -> Result<Vec<String>, Self::Err> {+ let mut state = lock(&self.state);+ state.workspace_calls.push(people);+ if state.fail_workspace_resolution {+ anyhow::bail!("workspace resolver unavailable");+ }+ Ok(state.resolved_workspace_ids.clone())+ }+}++impl WebhookRepo for MockRepository {+ type Err = anyhow::Error;++ async fn create_webhook(+ &self,+ _created_by_user_id: MacroUserIdStr<'static>,+ _workspace_id: String,+ _request: CreateWebhookRequest,+ _signing_secret: String,+ _headers: Value,+ ) -> Result<Webhook, Self::Err> {+ unimplemented!("not used by webhook event ingestion")+ }++ async fn get_webhook(&self, _webhook_id: String) -> Result<Option<Webhook>, Self::Err> {+ unimplemented!("not used by webhook event ingestion")+ }++ async fn list_active_webhooks_matching_event(+ &self,+ workspace_ids: Vec<String>,+ event_name: String,+ entity_id: String,+ ) -> Result<Vec<Webhook>, Self::Err> {+ let mut state = lock(&self.state);+ state.match_calls.push(MatchCall {+ workspace_ids,+ event_name,+ entity_id,+ });+ if state.fail_matching {+ anyhow::bail!("webhook repository unavailable");+ }+ Ok(state.matching_webhooks.clone())+ }++ async fn patch_webhook(+ &self,+ _webhook_id: String,+ _request: PatchWebhookRequest,+ ) -> Result<Option<Webhook>, Self::Err> {+ unimplemented!("not used by webhook event ingestion")+ }++ async fn delete_webhook(&self, _webhook_id: String) -> Result<Option<Webhook>, Self::Err> {+ unimplemented!("not used by webhook event ingestion")+ }++ async fn set_webhook_validity(+ &self,+ _webhook_id: String,+ _is_valid: bool,+ ) -> Result<Option<Webhook>, Self::Err> {+ unimplemented!("not used by webhook event ingestion")+ }++ async fn get_user_team_workspace_id(+ &self,+ _user_id: MacroUserIdStr<'static>,+ ) -> Result<Option<String>, Self::Err> {+ unimplemented!("not used by webhook event ingestion")+ }+}++#[derive(Default)]+struct MockEnqueuerState {+ attempted_messages: Vec<WebhookEventQueueMessage>,+ completed_webhook_ids: Vec<String>,+ failing_webhook_id: Option<String>,+ delayed_webhook_id: Option<String>,+}++#[derive(Clone, Default)]+struct MockEnqueuer {+ state: Arc<Mutex<MockEnqueuerState>>,+}++impl WebhookEventEnqueuer for MockEnqueuer {+ type Err = anyhow::Error;++ async fn enqueue(&self, message: WebhookEventQueueMessage) -> Result<(), Self::Err> {+ let (should_fail, should_delay) = {+ let mut state = lock(&self.state);+ let should_fail = state.failing_webhook_id.as_ref() == Some(&message.webhook_id);+ let should_delay = state.delayed_webhook_id.as_ref() == Some(&message.webhook_id);+ state.attempted_messages.push(message.clone());+ (should_fail, should_delay)+ };++ if should_delay {+ tokio::time::sleep(Duration::from_millis(10)).await;+ }+ lock(&self.state)+ .completed_webhook_ids+ .push(message.webhook_id.clone());++ if should_fail {+ anyhow::bail!("queue unavailable");+ }+ Ok(())+ }+}++type TestService =+ WebhookEventIngestionServiceImpl<MockAccessService, MockRepository, MockEnqueuer>;++fn service(+ access: MockAccessService,+ repository: MockRepository,+ enqueuer: MockEnqueuer,+) -> TestService {+ WebhookEventIngestionServiceImpl::new(Arc::new(access), repository, enqueuer)+}++fn webhook(id: &str, workspace_id: &str) -> Webhook {+ Webhook {+ id: id.to_string(),+ workspace_id: workspace_id.to_string(),+ name: id.to_string(),+ endpoint_url: "https://example.com/webhook".to_string(),+ signing_secret: "not-queued".to_string(),+ headers: BTreeMap::from([("X-Custom".to_string(), "not-queued".to_string())]),+ status: WebhookStatus::Active,+ is_valid: true,+ created_by_user_id: PERSONAL_WORKSPACE_ID.to_string(),+ created_at: timestamp(),+ updated_at: timestamp(),+ deleted_at: None,+ filters: Vec::new(),+ }+}++#[derive(Clone)]+enum TestBrokerEvent {+ Document(Event<DocumentTopicEvent>),+ Channel(Event<ChannelTopicEvent>),}-fn document_events() -> Vec<DocumentTopicEvent> {+impl TestBrokerEvent {+ async fn ingest(&self, service: &TestService) -> Result<(), WebhookEventIngestionError> {+ match self {+ Self::Document(event) => service.ingest_document_event(event.clone()).await,+ Self::Channel(event) => service.ingest_channel_event(event.clone()).await,+ }+ }++ fn envelope(&self) -> Value {+ match self {+ Self::Document(event) => serde_json::to_value(event).expect("serializable event"),+ Self::Channel(event) => serde_json::to_value(event).expect("serializable event"),+ }+ }++ fn event_id(&self) -> Uuid {+ match self {+ Self::Document(event) => event.event_id,+ Self::Channel(event) => event.event_id,+ }+ }++ fn schema_version(&self) -> u8 {+ match self {+ Self::Document(event) => event.schema_version,+ Self::Channel(event) => event.schema_version,+ }+ }+}++struct EventCase {+ event: TestBrokerEvent,+ event_name: &'static str,+ entity_type: EntityType,+ normalized_entity_type: &'static str,+ entity_id: String,+}++fn document_event_cases() -> Vec<EventCase> {vec![- DocumentTopicEvent::Created(DocumentCreatedMetadata {- document_id: "doc_1".to_string(),- owner: user_id("macro|[email protected]"),- document_name: "notes".to_string(),- file_type: None,- project_id: None,- sub_type: None,- created_at: Some(timestamp()),- }),- DocumentTopicEvent::Updated(DocumentUpdatedMetadata {- document_id: "doc_1".to_string(),- owner: user_id("macro|[email protected]"),- actor_user_id: Some(user_id("macro|[email protected]")),- document_name: Some("renamed".to_string()),- previous_project_id: None,- project_id: None,- file_type: None,- share_permission_updated: false,- }),- DocumentTopicEvent::Deleted(DocumentDeletedMetadata {- document_id: "doc_1".to_string(),- actor_user_id: Some(user_id("macro|[email protected]")),- project_id: None,- }),- DocumentTopicEvent::Copied(DocumentCopiedMetadata {- document_id: "doc_2".to_string(),- source_document_id: "doc_1".to_string(),- source_version_id: None,- owner: user_id("macro|[email protected]"),- document_name: "notes (copy)".to_string(),- file_type: None,- project_id: None,- sub_type: None,- }),+ EventCase {+ event: TestBrokerEvent::Document(Event::with_schema_version(+ DocumentTopicEvent::Created(DocumentCreatedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ owner: user_id("macro|[email protected]"),+ document_name: "notes".to_string(),+ file_type: None,+ project_id: None,+ sub_type: None,+ created_at: Some(timestamp()),+ }),+ 2,+ )),+ event_name: "document.created",+ entity_type: EntityType::Document,+ normalized_entity_type: DOCUMENT_ENTITY_TYPE,+ entity_id: DOCUMENT_ID.to_string(),+ },+ EventCase {+ event: TestBrokerEvent::Document(Event::with_schema_version(+ DocumentTopicEvent::Updated(DocumentUpdatedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ owner: user_id("macro|[email protected]"),+ actor_user_id: Some(user_id("macro|[email protected]")),+ document_name: Some("renamed".to_string()),+ previous_project_id: None,+ project_id: None,+ file_type: None,+ share_permission_updated: false,+ }),+ 2,+ )),+ event_name: "document.updated",+ entity_type: EntityType::Document,+ normalized_entity_type: DOCUMENT_ENTITY_TYPE,+ entity_id: DOCUMENT_ID.to_string(),+ },+ EventCase {+ event: TestBrokerEvent::Document(Event::with_schema_version(+ DocumentTopicEvent::Deleted(DocumentDeletedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ actor_user_id: Some(user_id("macro|[email protected]")),+ project_id: None,+ }),+ 2,+ )),+ event_name: "document.deleted",+ entity_type: EntityType::Document,+ normalized_entity_type: DOCUMENT_ENTITY_TYPE,+ entity_id: DOCUMENT_ID.to_string(),+ },+ EventCase {+ event: TestBrokerEvent::Document(Event::with_schema_version(+ DocumentTopicEvent::Copied(DocumentCopiedMetadata {+ document_id: COPIED_DOCUMENT_ID.to_string(),+ source_document_id: DOCUMENT_ID.to_string(),+ source_version_id: None,+ owner: user_id("macro|[email protected]"),+ document_name: "notes (copy)".to_string(),+ file_type: None,+ project_id: None,+ sub_type: None,+ }),+ 2,+ )),+ event_name: "document.copied",+ entity_type: EntityType::Document,+ normalized_entity_type: DOCUMENT_ENTITY_TYPE,+ entity_id: COPIED_DOCUMENT_ID.to_string(),+ },]}-fn channel_events() -> Vec<ChannelTopicEvent> {- let channel_id = Uuid::nil();- let message_id = Uuid::nil();+fn channel_event_cases() -> Vec<EventCase> {+ let channel_id = Uuid::parse_str("44444444-4444-4444-4444-444444444444").unwrap();+ let message_id = Uuid::parse_str("55555555-5555-5555-5555-555555555555").unwrap();+ let owner = "macro|[email protected]";+ let member = "macro|[email protected]";+vec![- ChannelTopicEvent::Created(ChannelCreatedMetadata {- channel_id,- actor: sender("macro|[email protected]"),- channel_type: ChannelType::Team,- channel_name: Some("general".to_string()),- participant_user_ids: vec![user_id("macro|[email protected]")],- }),- ChannelTopicEvent::Updated(ChannelUpdatedMetadata {- channel_id,- actor: user_id("macro|[email protected]"),- previous_name: Some("general".to_string()),- channel_name: Some("announcements".to_string()),- }),- ChannelTopicEvent::Deleted(ChannelDeletedMetadata {- channel_id,- actor: sender("macro|[email protected]"),- }),- ChannelTopicEvent::MessagePosted(ChannelMessagePostedMetadata {- channel_id,- message_id,- thread_id: None,- sender: sender("macro|[email protected]"),- triggered_by: None,- channel_type: ChannelType::Team,- content: "hello".to_string(),- mentions: vec![],- attachments: vec![],- created_at: timestamp(),- }),- ChannelTopicEvent::MessagePatched(ChannelMessagePatchedMetadata {- channel_id,- message_id,- thread_id: None,- actor: sender("macro|[email protected]"),- content: "hello (edited)".to_string(),- edited_at: Some(timestamp()),- updated_at: timestamp(),- }),- ChannelTopicEvent::MessageDeleted(ChannelMessageDeletedMetadata {- channel_id,- message_id,- thread_id: None,- actor: sender("macro|[email protected]"),- deleted_at: Some(timestamp()),- }),- ChannelTopicEvent::MessageAttachmentCreated(ChannelMessageAttachmentCreatedMetadata {- channel_id,- message_id,- actor: sender("macro|[email protected]"),- attachments: vec![],- }),- ChannelTopicEvent::MessageAttachmentRemoved(ChannelMessageAttachmentRemovedMetadata {- channel_id,- message_id,- actor: sender("macro|[email protected]"),- attachments: vec![],- }),- ChannelTopicEvent::ParticipantAdded(ChannelParticipantAddedMetadata {- channel_id,- channel_type: ChannelType::Team,- added_by: sender("macro|[email protected]"),- added_user_ids: vec![user_id("macro|[email protected]")],- }),- ChannelTopicEvent::ParticipantRemoved(ChannelParticipantRemovedMetadata {- channel_id,- channel_type: ChannelType::Team,- removed_by: user_id("macro|[email protected]"),- removed_user_ids: vec![user_id("macro|[email protected]")],- }),+ EventCase {+ event: TestBrokerEvent::Channel(Event::with_schema_version(+ ChannelTopicEvent::Created(ChannelCreatedMetadata {+ channel_id,+ actor: sender(owner),+ channel_type: ChannelType::Team,+ channel_name: Some("general".to_string()),+ participant_user_ids: vec![user_id(owner)],+ }),+ 3,+ )),+ event_name: "channel.created",+ entity_type: EntityType::Channel,+ normalized_entity_type: CHANNEL_ENTITY_TYPE,+ entity_id: channel_id.to_string(),+ },+ EventCase {+ event: TestBrokerEvent::Channel(Event::with_schema_version(+ ChannelTopicEvent::Updated(ChannelUpdatedMetadata {+ channel_id,+ actor: user_id(owner),+ previous_name: Some("general".to_string()),+ channel_name: Some("announcements".to_string()),+ }),+ 3,+ )),+ event_name: "channel.updated",+ entity_type: EntityType::Channel,+ normalized_entity_type: CHANNEL_ENTITY_TYPE,+ entity_id: channel_id.to_string(),+ },+ EventCase {+ event: TestBrokerEvent::Channel(Event::with_schema_version(+ ChannelTopicEvent::Deleted(ChannelDeletedMetadata {+ channel_id,+ actor: sender(owner),+ }),+ 3,+ )),+ event_name: "channel.deleted",+ entity_type: EntityType::Channel,+ normalized_entity_type: CHANNEL_ENTITY_TYPE,+ entity_id: channel_id.to_string(),+ },+ EventCase {+ event: TestBrokerEvent::Channel(Event::with_schema_version(+ ChannelTopicEvent::MessagePosted(ChannelMessagePostedMetadata {+ channel_id,+ message_id,+ thread_id: None,+ sender: sender(member),+ triggered_by: None,+ channel_type: ChannelType::Team,+ content: "hello".to_string(),+ mentions: vec![],+ attachments: vec![],+ created_at: timestamp(),+ }),+ 3,+ )),+ event_name: "channel.message_posted",+ entity_type: EntityType::Channel,+ normalized_entity_type: CHANNEL_ENTITY_TYPE,+ entity_id: channel_id.to_string(),+ },+ EventCase {+ event: TestBrokerEvent::Channel(Event::with_schema_version(+ ChannelTopicEvent::MessagePatched(ChannelMessagePatchedMetadata {+ channel_id,+ message_id,+ thread_id: None,+ actor: sender(member),+ content: "hello (edited)".to_string(),+ edited_at: Some(timestamp()),+ updated_at: timestamp(),+ }),+ 3,+ )),+ event_name: "channel.message_patched",+ entity_type: EntityType::Channel,+ normalized_entity_type: CHANNEL_ENTITY_TYPE,+ entity_id: channel_id.to_string(),+ },+ EventCase {+ event: TestBrokerEvent::Channel(Event::with_schema_version(+ ChannelTopicEvent::MessageDeleted(ChannelMessageDeletedMetadata {+ channel_id,+ message_id,+ thread_id: None,+ actor: sender(member),+ deleted_at: Some(timestamp()),+ }),+ 3,+ )),+ event_name: "channel.message_deleted",+ entity_type: EntityType::Channel,+ normalized_entity_type: CHANNEL_ENTITY_TYPE,+ entity_id: channel_id.to_string(),+ },+ EventCase {+ event: TestBrokerEvent::Channel(Event::with_schema_version(+ ChannelTopicEvent::MessageAttachmentCreated(+ ChannelMessageAttachmentCreatedMetadata {+ channel_id,+ message_id,+ actor: sender(member),+ attachments: vec![],+ },+ ),+ 3,+ )),+ event_name: "channel.message_attachment_created",+ entity_type: EntityType::Channel,+ normalized_entity_type: CHANNEL_ENTITY_TYPE,+ entity_id: channel_id.to_string(),+ },+ EventCase {+ event: TestBrokerEvent::Channel(Event::with_schema_version(+ ChannelTopicEvent::MessageAttachmentRemoved(+ ChannelMessageAttachmentRemovedMetadata {+ channel_id,+ message_id,+ actor: sender(member),+ attachments: vec![],+ },+ ),+ 3,+ )),+ event_name: "channel.message_attachment_removed",+ entity_type: EntityType::Channel,+ normalized_entity_type: CHANNEL_ENTITY_TYPE,+ entity_id: channel_id.to_string(),+ },+ EventCase {+ event: TestBrokerEvent::Channel(Event::with_schema_version(+ ChannelTopicEvent::ParticipantAdded(ChannelParticipantAddedMetadata {+ channel_id,+ channel_type: ChannelType::Team,+ added_by: sender(owner),+ added_user_ids: vec![user_id(member)],+ }),+ 3,+ )),+ event_name: "channel.participant_added",+ entity_type: EntityType::Channel,+ normalized_entity_type: CHANNEL_ENTITY_TYPE,+ entity_id: channel_id.to_string(),+ },+ EventCase {+ event: TestBrokerEvent::Channel(Event::with_schema_version(+ ChannelTopicEvent::ParticipantRemoved(ChannelParticipantRemovedMetadata {+ channel_id,+ channel_type: ChannelType::Team,+ removed_by: user_id(owner),+ removed_user_ids: vec![user_id(member)],+ }),+ 3,+ )),+ event_name: "channel.participant_removed",+ entity_type: EntityType::Channel,+ normalized_entity_type: CHANNEL_ENTITY_TYPE,+ entity_id: channel_id.to_string(),+ },]}#[tokio::test]-async fn ingests_every_document_event_variant() {- let service = service();- for event in document_events() {- service- .ingest_document_event(Event::new(event))+async fn normalizes_and_matches_all_fourteen_event_variants() {+ let event_cases = document_event_cases()+ .into_iter()+ .chain(channel_event_cases())+ .collect::<Vec<_>>();+ assert_eq!(event_cases.len(), 14);++ for event_case in event_cases {+ let access = MockAccessService::with_users(vec![user_id(PERSONAL_WORKSPACE_ID)]);+ let repository = MockRepository::new(+ vec![PERSONAL_WORKSPACE_ID.to_string()],+ vec![webhook("wh_match", PERSONAL_WORKSPACE_ID)],+ );+ let enqueuer = MockEnqueuer::default();+ let service = service(access.clone(), repository.clone(), enqueuer.clone());+ let expected_envelope = event_case.event.envelope();+ let before_ingestion = Utc::now();++ event_case+ .event+ .ingest(&service).await- .expect("stub handlers always succeed");+ .unwrap_or_else(|error| {+ panic!("{} should be ingested: {error}", event_case.event_name)+ });+ let after_ingestion = Utc::now();++ assert_eq!(+ lock(&access.calls).as_slice(),+ &[(event_case.entity_id.clone(), event_case.entity_type)],+ "entity access mapping for {}",+ event_case.event_name+ );+ let repository_state = lock(&repository.state);+ assert_eq!(+ repository_state.match_calls.as_slice(),+ &[MatchCall {+ workspace_ids: vec![PERSONAL_WORKSPACE_ID.to_string()],+ event_name: event_case.event_name.to_string(),+ entity_id: event_case.entity_id.clone(),+ }],+ "repository matching for {}",+ event_case.event_name+ );+ drop(repository_state);++ let enqueuer_state = lock(&enqueuer.state);+ assert_eq!(enqueuer_state.attempted_messages.len(), 1);+ let message = &enqueuer_state.attempted_messages[0];+ assert_eq!(message.webhook_id, "wh_match");+ assert_eq!(+ message.event.event_id,+ event_case.event.event_id().to_string()+ );+ assert_eq!(+ message.event.schema_version,+ event_case.event.schema_version()+ );+ assert_eq!(message.event.event_name, event_case.event_name);+ assert_eq!(message.event.entity_type, event_case.normalized_entity_type);+ assert_eq!(message.event.entity_id, event_case.entity_id);+ assert_eq!(message.event.ordering_key, event_case.entity_id);+ assert_eq!(message.event.broker_envelope, expected_envelope);+ assert_eq!(+ message.event.broker_envelope["event_type"],+ event_case.event_name+ );+ assert!(message.event.occurred_at >= before_ingestion);+ assert!(message.event.occurred_at <= after_ingestion);}}#[tokio::test]-async fn ingests_every_channel_event_variant() {- let service = service();- for event in channel_events() {- service- .ingest_channel_event(Event::new(event))- .await- .expect("stub handlers always succeed");+async fn no_accessors_resolves_no_workspaces_and_enqueues_nothing() {+ let access = MockAccessService::with_users(Vec::new());+ let repository = MockRepository::new(Vec::new(), Vec::new());+ let enqueuer = MockEnqueuer::default();+ let service = service(access, repository.clone(), enqueuer.clone());+ let event = document_event_cases().remove(0).event;++ event.ingest(&service).await.expect("no access is a no-op");++ let repository_state = lock(&repository.state);+ assert_eq!(repository_state.workspace_calls, vec![Vec::new()]);+ assert_eq!(+ repository_state.match_calls,+ vec![MatchCall {+ workspace_ids: Vec::new(),+ event_name: "document.created".to_string(),+ entity_id: DOCUMENT_ID.to_string(),+ }]+ );+ assert!(lock(&enqueuer.state).attempted_messages.is_empty());+}++#[tokio::test]+async fn no_matching_webhooks_enqueues_nothing_after_workspace_expansion() {+ let reader = user_id(PERSONAL_WORKSPACE_ID);+ let access = MockAccessService::with_users(vec![reader.clone()]);+ let repository = MockRepository::new(+ vec![+ PERSONAL_WORKSPACE_ID.to_string(),+ TEAM_WORKSPACE_ID.to_string(),+ ],+ Vec::new(),+ );+ let enqueuer = MockEnqueuer::default();+ let service = service(access, repository.clone(), enqueuer.clone());+ let event = document_event_cases().remove(0).event;++ event.ingest(&service).await.expect("no matches is a no-op");++ let repository_state = lock(&repository.state);+ assert_eq!(repository_state.workspace_calls, vec![vec![reader]]);+ assert_eq!(+ repository_state.match_calls[0].workspace_ids,+ vec![+ PERSONAL_WORKSPACE_ID.to_string(),+ TEAM_WORKSPACE_ID.to_string(),+ ]+ );+ assert!(lock(&enqueuer.state).attempted_messages.is_empty());+}++#[tokio::test]+async fn fans_out_exact_envelope_once_per_personal_and_team_webhook() {+ let event = document_event_cases().remove(3).event;+ let expected_envelope = event.envelope();+ let reader = user_id(PERSONAL_WORKSPACE_ID);+ let access = MockAccessService::with_users(vec![reader.clone()]);+ let repository = MockRepository::new(+ vec![+ PERSONAL_WORKSPACE_ID.to_string(),+ TEAM_WORKSPACE_ID.to_string(),+ ],+ vec![+ webhook("wh_personal", PERSONAL_WORKSPACE_ID),+ webhook("wh_team", TEAM_WORKSPACE_ID),+ ],+ );+ let enqueuer = MockEnqueuer::default();+ let service = service(access, repository.clone(), enqueuer.clone());++ event.ingest(&service).await.expect("fan-out succeeds");++ let repository_state = lock(&repository.state);+ assert_eq!(repository_state.workspace_calls, vec![vec![reader]]);+ assert_eq!(+ repository_state.match_calls,+ vec![MatchCall {+ workspace_ids: vec![+ PERSONAL_WORKSPACE_ID.to_string(),+ TEAM_WORKSPACE_ID.to_string(),+ ],+ event_name: "document.copied".to_string(),+ entity_id: COPIED_DOCUMENT_ID.to_string(),+ }]+ );+ drop(repository_state);++ let enqueuer_state = lock(&enqueuer.state);+ let mut webhook_ids = enqueuer_state+ .attempted_messages+ .iter()+ .map(|message| message.webhook_id.as_str())+ .collect::<Vec<_>>();+ webhook_ids.sort_unstable();+ assert_eq!(webhook_ids, vec!["wh_personal", "wh_team"]);+ for message in &enqueuer_state.attempted_messages {+ assert_eq!(message.event.broker_envelope, expected_envelope);+ assert_eq!(message.event.entity_id, COPIED_DOCUMENT_ID);+ }+}++#[tokio::test]+async fn enqueue_failure_is_transient_after_every_send_completes() {+ let access = MockAccessService::with_users(vec![user_id(PERSONAL_WORKSPACE_ID)]);+ let repository = MockRepository::new(+ vec![PERSONAL_WORKSPACE_ID.to_string()],+ vec![+ webhook("wh_fail", PERSONAL_WORKSPACE_ID),+ webhook("wh_slow", PERSONAL_WORKSPACE_ID),+ ],+ );+ let enqueuer = MockEnqueuer::default();+ {+ let mut state = lock(&enqueuer.state);+ state.failing_webhook_id = Some("wh_fail".to_string());+ state.delayed_webhook_id = Some("wh_slow".to_string());}+ let service = service(access, repository, enqueuer.clone());+ let event = document_event_cases().remove(0).event;++ let error = event+ .ingest(&service)+ .await+ .expect_err("one failed send fails ingestion");++ assert!(matches!(error, WebhookEventIngestionError::Enqueue(_)));+ assert!(error.is_transient());+ let mut completed_webhook_ids = lock(&enqueuer.state).completed_webhook_ids.clone();+ completed_webhook_ids.sort_unstable();+ assert_eq!(completed_webhook_ids, vec!["wh_fail", "wh_slow"]);+}++#[tokio::test]+async fn malformed_document_id_is_permanent_and_skips_access_resolution() {+ let access = MockAccessService::with_users(vec![user_id(PERSONAL_WORKSPACE_ID)]);+ let repository = MockRepository::new(Vec::new(), Vec::new());+ let enqueuer = MockEnqueuer::default();+ let service = service(access.clone(), repository, enqueuer);+ let event = Event::new(DocumentTopicEvent::Deleted(DocumentDeletedMetadata {+ document_id: "not-a-uuid".to_string(),+ actor_user_id: None,+ project_id: None,+ }));++ let error = service+ .ingest_document_event(event)+ .await+ .expect_err("malformed ids are rejected");++ assert!(matches!(+ error,+ WebhookEventIngestionError::InvalidEntityId { .. }+ ));+ assert!(!error.is_transient());+ assert!(lock(&access.calls).is_empty());}#[test]-fn classifies_transient_errors() {+fn classifies_adapter_and_contract_errors() {let database_error = WebhookEventIngestionError::EntityAccess(AccessError::DatabaseError(sqlx::Error::PoolTimedOut,));assert!(database_error.is_transient());+ assert!(WebhookEventIngestionError::EntityAccess(AccessError::Internal).is_transient());+ assert!(!WebhookEventIngestionError::EntityAccess(AccessError::Unauthorized).is_transient());- let unauthorized = WebhookEventIngestionError::EntityAccess(AccessError::Unauthorized);- assert!(!unauthorized.is_transient());-- let internal = WebhookEventIngestionError::Internal(anyhow::anyhow!("adapter bug"));- assert!(!internal.is_transient());+ let serialization_error = serde_json::from_str::<Value>("{").expect_err("invalid json");+ assert!(!WebhookEventIngestionError::Serialization(serialization_error).is_transient());+ assert!(WebhookEventIngestionError::WorkspaceResolution(anyhow::anyhow!("db")).is_transient());+ assert!(WebhookEventIngestionError::Repository(anyhow::anyhow!("db")).is_transient());+ assert!(WebhookEventIngestionError::Enqueue(anyhow::anyhow!("queue")).is_transient());}#[tokio::test]-async fn users_with_access_delegates_to_entity_access() {- let users = service()- .users_with_access("doc_1", EntityType::Document)+async fn entity_access_internal_error_is_transient() {+ let access = MockAccessService {+ users: Vec::new(),+ failure: Some(AccessFailure::Internal),+ calls: Arc::default(),+ };+ let repository = MockRepository::new(Vec::new(), Vec::new());+ let enqueuer = MockEnqueuer::default();+ let service = service(access, repository, enqueuer);+ let event = document_event_cases().remove(0).event;++ let error = event+ .ingest(&service).await- .expect("mock resolves users");- assert_eq!(users, vec![user_id("macro|[email protected]")]);+ .expect_err("access adapter fails");++ assert!(matches!(+ error,+ WebhookEventIngestionError::EntityAccess(AccessError::Internal)+ ));+ assert!(error.is_transient());}diff --git a/rust/cloud-storage/webhook/src/domain/mod.rs b/rust/cloud-storage/webhook/src/domain/mod.rsindex ea2b4e5c8f..79230450ad 100644--- a/rust/cloud-storage/webhook/src/domain/mod.rs+++ b/rust/cloud-storage/webhook/src/domain/mod.rs@@ -1,5 +1,8 @@//! Domain layer for webhooks.+#[cfg(feature = "ports")]+/// Webhook event delivery state machine and retry policy.+pub mod delivery;#[cfg(feature = "ingestion")]/// Webhook event ingestion service.pub mod ingestion;diff --git a/rust/cloud-storage/webhook/src/domain/models.rs b/rust/cloud-storage/webhook/src/domain/models.rsindex e2edd489fd..f0ddf469bb 100644--- a/rust/cloud-storage/webhook/src/domain/models.rs+++ b/rust/cloud-storage/webhook/src/domain/models.rs@@ -2,7 +2,7 @@use chrono::{DateTime, Utc};use serde::{Deserialize, Serialize};-use std::collections::BTreeMap;+use std::{collections::BTreeMap, time::Duration};/// Webhook id. Webhook ids are stored with a `wh_` prefix.pub type WebhookId = String;@@ -10,6 +10,261 @@ pub type WebhookId = String;/// Custom headers supplied for webhook delivery.pub type WebhookHeaders = BTreeMap<String, String>;+/// Version of the webhook event queue message contract emitted by this crate.+#[cfg(feature = "ports")]+pub const WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION: u8 = 1;++/// Event data normalized from a broker envelope for webhook matching and delivery.+///+/// `broker_envelope` retains the complete payload received from the broker and is+/// the only value sent as the webhook HTTP request body.+#[cfg(feature = "ports")]+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct NormalizedWebhookEvent {+ /// Broker event identifier, represented as a string for transport neutrality.+ pub event_id: String,+ /// Version of the broker event payload schema.+ pub schema_version: u8,+ /// Exact event name used for webhook filter matching.+ pub event_name: String,+ /// Normalized entity type used for access resolution and delivery records.+ pub entity_type: String,+ /// Entity identifier used for access resolution and webhook filter matching.+ pub entity_id: String,+ /// Key that preserves the event ordering observed during ingestion.+ pub ordering_key: String,+ /// Time at which the broker event was ingested for webhook processing.+ pub occurred_at: DateTime<Utc>,+ /// Complete broker envelope forwarded as the webhook HTTP request body.+ pub broker_envelope: serde_json::Value,+}++/// Versioned queue payload for delivering one event to one webhook.+///+/// Endpoint URLs, custom headers, and signing secrets are deliberately absent;+/// workers must resolve the webhook's current configuration before delivery.+#[cfg(feature = "ports")]+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]+#[serde(deny_unknown_fields)]+pub struct WebhookEventQueueMessage {+ /// Queue contract version.+ pub version: u8,+ /// Webhook that should receive the event.+ pub webhook_id: WebhookId,+ /// Normalized broker event to deliver.+ pub event: NormalizedWebhookEvent,+}++#[cfg(feature = "ports")]+impl WebhookEventQueueMessage {+ /// Create a queue message using the current contract version.+ pub fn new(webhook_id: WebhookId, event: NormalizedWebhookEvent) -> Self {+ Self {+ version: WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION,+ webhook_id,+ event,+ }+ }++ /// Return whether this message uses a contract version supported by this crate.+ pub fn has_supported_version(&self) -> bool {+ self.version == WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION+ }+}++/// Raw queue message retained at the queue adapter boundary.+///+/// The optional body and receipt handle are kept separate so a worker can+/// acknowledge malformed or missing payloads whenever the queue supplied a+/// receipt handle.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct RawWebhookEventQueueMessage {+ /// Queue-assigned message identifier, when supplied by the queue.+ pub message_id: Option<String>,+ /// Unparsed queue message body.+ pub body: Option<String>,+ /// Handle used to acknowledge or delay this receipt.+ pub receipt_handle: Option<String>,+}++/// Identifier of a persisted webhook delivery.+pub type WebhookDeliveryId = String;++/// Identifier of a persisted webhook delivery attempt.+pub type WebhookDeliveryAttemptId = String;++/// Persisted lifecycle state of a webhook delivery.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum WebhookDeliveryStatus {+ /// Delivery is persisted and ready for its first attempt.+ Queued,+ /// An HTTP attempt has started and has not completed.+ InProgress,+ /// A retryable attempt failed and another attempt is scheduled.+ RetryScheduled,+ /// The endpoint accepted the event.+ Delivered,+ /// Delivery ended after a non-retryable failure.+ PermanentlyFailed,+ /// Delivery exhausted the configured HTTP attempt limit.+ Exhausted,+ /// Delivery was canceled because the webhook is no longer eligible.+ Canceled,+}++impl WebhookDeliveryStatus {+ /// Storage representation.+ pub const fn as_str(self) -> &'static str {+ match self {+ Self::Queued => "queued",+ Self::InProgress => "in_progress",+ Self::RetryScheduled => "retry_scheduled",+ Self::Delivered => "delivered",+ Self::PermanentlyFailed => "permanently_failed",+ Self::Exhausted => "exhausted",+ Self::Canceled => "canceled",+ }+ }++ /// Return whether this status prevents any further HTTP attempts.+ pub const fn is_terminal(self) -> bool {+ matches!(+ self,+ Self::Delivered | Self::PermanentlyFailed | Self::Exhausted | Self::Canceled+ )+ }+}++impl std::str::FromStr for WebhookDeliveryStatus {+ type Err = String;++ fn from_str(value: &str) -> Result<Self, Self::Err> {+ match value {+ "queued" => Ok(Self::Queued),+ "in_progress" => Ok(Self::InProgress),+ "retry_scheduled" => Ok(Self::RetryScheduled),+ "delivered" => Ok(Self::Delivered),+ "permanently_failed" => Ok(Self::PermanentlyFailed),+ "exhausted" => Ok(Self::Exhausted),+ "canceled" => Ok(Self::Canceled),+ other => Err(format!("unknown webhook delivery status: {other}")),+ }+ }+}++/// Persisted lifecycle state of one webhook delivery attempt.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum WebhookDeliveryAttemptStatus {+ /// The HTTP request has started and has not completed.+ InProgress,+ /// The endpoint accepted the request.+ Succeeded,+ /// The request failed with a retryable outcome.+ RetryableFailure,+ /// The request failed with a permanent outcome.+ PermanentFailure,+ /// The request failed and exhausted the delivery's attempt limit.+ Exhausted,+ /// Processing stopped before the attempt outcome could be recorded.+ Interrupted,+}++impl WebhookDeliveryAttemptStatus {+ /// Storage representation.+ pub const fn as_str(self) -> &'static str {+ match self {+ Self::InProgress => "in_progress",+ Self::Succeeded => "succeeded",+ Self::RetryableFailure => "retryable_failure",+ Self::PermanentFailure => "permanent_failure",+ Self::Exhausted => "exhausted",+ Self::Interrupted => "interrupted",+ }+ }+}++impl std::str::FromStr for WebhookDeliveryAttemptStatus {+ type Err = String;++ fn from_str(value: &str) -> Result<Self, Self::Err> {+ match value {+ "in_progress" => Ok(Self::InProgress),+ "succeeded" => Ok(Self::Succeeded),+ "retryable_failure" => Ok(Self::RetryableFailure),+ "permanent_failure" => Ok(Self::PermanentFailure),+ "exhausted" => Ok(Self::Exhausted),+ "interrupted" => Ok(Self::Interrupted),+ other => Err(format!("unknown webhook delivery attempt status: {other}")),+ }+ }+}++/// Transport-neutral metadata captured from one webhook HTTP request.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct WebhookHttpOutcomeDetails {+ /// Time spent making the HTTP request.+ pub duration: Duration,+ /// HTTP response status, when the endpoint returned a response.+ pub response_status: Option<u16>,+ /// Response header names with values redacted by the delivery client.+ pub response_headers_redacted: Option<WebhookHeaders>,+ /// Lossy response-body preview, capped by the delivery client.+ pub response_body_preview: Option<String>,+ /// Sanitized failure category, when the request failed.+ pub error_kind: Option<String>,+ /// Sanitized failure message, when the request failed.+ pub error_message: Option<String>,+}++/// Classification and metadata returned by a webhook HTTP delivery client.+#[derive(Debug, Clone, PartialEq, Eq)]+pub enum WebhookHttpOutcome {+ /// The endpoint returned a successful response.+ Success(WebhookHttpOutcomeDetails),+ /// The request may succeed if attempted again.+ RetryableFailure(WebhookHttpOutcomeDetails),+ /// The request must not be attempted again.+ PermanentFailure(WebhookHttpOutcomeDetails),+}++/// Current persisted delivery and webhook configuration loaded idempotently.+#[derive(Debug, Clone)]+pub struct PreparedWebhookDelivery {+ /// Persisted delivery identifier.+ pub delivery_id: WebhookDeliveryId,+ /// Current webhook configuration, including outbound-only signing material.+ pub webhook: Webhook,+ /// Current persisted delivery status.+ pub status: WebhookDeliveryStatus,+ /// Number of HTTP attempts that have already started.+ pub attempt_count: u32,+ /// Earliest time at which another attempt may begin.+ pub next_attempt_at: Option<DateTime<Utc>>,+}++/// Numbered webhook delivery attempt started by the repository.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct WebhookDeliveryAttempt {+ /// Persisted attempt identifier.+ pub attempt_id: WebhookDeliveryAttemptId,+ /// Delivery to which this attempt belongs.+ pub delivery_id: WebhookDeliveryId,+ /// One-based attempt number.+ pub attempt_number: u32,+}++/// Action the inbound worker should apply to a processed queue receipt.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum WebhookWorkerDisposition {+ /// Delete the queue receipt.+ Acknowledge,+ /// Make the queue receipt available again after the supplied delay.+ RetryAfter(Duration),+}+/// Event and optional entity-id constraints used to match webhook deliveries.#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]#[cfg_attr(feature = "inbound", derive(utoipa::ToSchema))]@@ -73,6 +328,28 @@ pub enum WebhookScope {Team,}+/// Endpoint restrictions applied to webhook URLs.+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]+pub enum WebhookEndpointSchemePolicy {+ /// Require HTTPS endpoints with public addresses.+ #[default]+ HttpsOnly,+ /// Allow HTTP and non-public addresses for local testing.+ HttpAndHttps,+}++impl WebhookEndpointSchemePolicy {+ /// Return whether the URL scheme is permitted by this policy.+ pub fn allows(self, scheme: &str) -> bool {+ scheme == "https" || (self == Self::HttpAndHttps && scheme == "http")+ }++ /// Return whether localhost, loopback, private, and link-local addresses are permitted.+ pub fn allows_local_addresses(self) -> bool {+ self == Self::HttpAndHttps+ }+}+/// Request to create a webhook.#[derive(Debug, Clone, Deserialize)]#[cfg_attr(feature = "inbound", derive(utoipa::ToSchema))]@@ -81,7 +358,7 @@ pub struct CreateWebhookRequest {pub scope: WebhookScope,/// Display name.pub name: String,- /// HTTPS endpoint URL.+ /// Endpoint URL. HTTPS is required outside local environments.pub endpoint_url: String,/// Optional custom delivery headers.pub headers: Option<WebhookHeaders>,@@ -95,7 +372,7 @@ pub struct CreateWebhookRequest {pub struct PatchWebhookRequest {/// Display name.pub name: Option<String>,- /// HTTPS endpoint URL.+ /// Endpoint URL. HTTPS is required outside local environments.pub endpoint_url: Option<String>,/// Optional custom delivery headers. When present, replaces existing headers.pub headers: Option<WebhookHeaders>,@@ -115,7 +392,7 @@ pub struct Webhook {pub workspace_id: String,/// Display name.pub name: String,- /// HTTPS endpoint URL.+ /// Endpoint URL. HTTPS is required outside local environments.pub endpoint_url: String,/// Signing secret used by outbound adapters. This is never serialized by APIs.#[serde(default, skip_serializing)]diff --git a/rust/cloud-storage/webhook/src/domain/models_test.rs b/rust/cloud-storage/webhook/src/domain/models_test.rsindex 5e1154d331..126f475b38 100644--- a/rust/cloud-storage/webhook/src/domain/models_test.rs+++ b/rust/cloud-storage/webhook/src/domain/models_test.rs@@ -1,4 +1,10 @@+#[cfg(feature = "ports")]+use super::models::{+ NormalizedWebhookEvent, WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION, WebhookEventQueueMessage,+};use super::models::{WebhookFilter, WebhookFilters};+#[cfg(feature = "ports")]+use chrono::{DateTime, Utc};use serde_json::json;#[test]@@ -74,3 +80,78 @@ fn rejects_non_string_array_entries() {assert!(result.is_err());}}++#[cfg(feature = "ports")]+#[test]+fn queue_message_json_round_trip_retains_normalized_event() {+ let broker_envelope = json!({+ "event_id": "019bbb3e-8d80-7000-8000-000000000001",+ "schema_version": 2,+ "event_type": "document.created",+ "metadata": {+ "document_id": "doc_123",+ "name": "Quarterly plan",+ },+ });+ let occurred_at = DateTime::parse_from_rfc3339("2026-07-10T15:30:00Z")+ .unwrap()+ .with_timezone(&Utc);+ let message = WebhookEventQueueMessage::new(+ "wh_123".to_string(),+ NormalizedWebhookEvent {+ event_id: "019bbb3e-8d80-7000-8000-000000000001".to_string(),+ schema_version: 2,+ event_name: "document.created".to_string(),+ entity_type: "document".to_string(),+ entity_id: "doc_123".to_string(),+ ordering_key: "doc_123".to_string(),+ occurred_at,+ broker_envelope: broker_envelope.clone(),+ },+ );++ let serialized = serde_json::to_value(&message).unwrap();+ assert_eq!(+ serialized,+ json!({+ "version": WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION,+ "webhook_id": "wh_123",+ "event": {+ "event_id": "019bbb3e-8d80-7000-8000-000000000001",+ "schema_version": 2,+ "event_name": "document.created",+ "entity_type": "document",+ "entity_id": "doc_123",+ "ordering_key": "doc_123",+ "occurred_at": "2026-07-10T15:30:00Z",+ "broker_envelope": broker_envelope,+ },+ })+ );++ let round_trip: WebhookEventQueueMessage = serde_json::from_value(serialized).unwrap();+ assert_eq!(round_trip, message);+ assert!(round_trip.has_supported_version());+}++#[cfg(feature = "ports")]+#[test]+fn queue_message_rejects_delivery_configuration() {+ let result = serde_json::from_value::<WebhookEventQueueMessage>(json!({+ "version": WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION,+ "webhook_id": "wh_123",+ "endpoint_url": "https://example.com/webhook",+ "event": {+ "event_id": "evt_123",+ "schema_version": 1,+ "event_name": "document.created",+ "entity_type": "document",+ "entity_id": "doc_123",+ "ordering_key": "doc_123",+ "occurred_at": "2026-07-10T15:30:00Z",+ "broker_envelope": {},+ },+ }));++ assert!(result.is_err());+}diff --git a/rust/cloud-storage/webhook/src/domain/ports.rs b/rust/cloud-storage/webhook/src/domain/ports.rsindex 09507ff053..95bb57b3db 100644--- a/rust/cloud-storage/webhook/src/domain/ports.rs+++ b/rust/cloud-storage/webhook/src/domain/ports.rs@@ -1,11 +1,14 @@//! Webhook domain ports.use super::models::{- CreateWebhookRequest, PatchWebhookRequest, ValidateWebhookResponse, Webhook, WebhookId,- WebhookValidationResult,+ CreateWebhookRequest, NormalizedWebhookEvent, PatchWebhookRequest, PreparedWebhookDelivery,+ RawWebhookEventQueueMessage, ValidateWebhookResponse, Webhook, WebhookDeliveryAttempt,+ WebhookEventQueueMessage, WebhookHttpOutcome, WebhookHttpOutcomeDetails, WebhookId,+ WebhookValidationResult, WebhookWorkerDisposition,};+use chrono::{DateTime, Utc};use macro_user_id::user_id::MacroUserIdStr;-use std::future::Future;+use std::{future::Future, time::Duration};/// Webhook repository.pub trait WebhookRepo: Clone + Send + Sync + 'static {@@ -66,6 +69,138 @@ pub trait WebhookRepo: Clone + Send + Sync + 'static {) -> impl Future<Output = Result<Option<String>, Self::Err>> + Send;}+/// Resolver that expands people into personal and team webhook workspace ids.+pub trait WebhookWorkspaceResolver: Clone + Send + Sync + 'static {+ /// Resolver error.+ type Err: Into<anyhow::Error> + Send;++ /// Resolve a deterministic, deduplicated set of personal and team workspace ids.+ fn resolve_workspace_ids(+ &self,+ people: Vec<MacroUserIdStr<'static>>,+ ) -> impl Future<Output = Result<Vec<String>, Self::Err>> + Send;+}++/// Port for enqueueing one normalized event for one matched webhook.+pub trait WebhookEventEnqueuer: Clone + Send + Sync + 'static {+ /// Enqueuer error.+ type Err: Into<anyhow::Error> + Send;++ /// Enqueue one webhook event delivery.+ fn enqueue(+ &self,+ message: WebhookEventQueueMessage,+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;+}++/// Queue operations used by the inbound webhook event worker.+pub trait WebhookEventQueue: Clone + Send + Sync + 'static {+ /// Queue error.+ type Err: Into<anyhow::Error> + Send;++ /// Receive a batch of unparsed queue messages.+ fn receive_messages(+ &self,+ ) -> impl Future<Output = Result<Vec<RawWebhookEventQueueMessage>, Self::Err>> + Send;++ /// Delete a queue receipt after processing or poison-message handling.+ fn delete_message(+ &self,+ receipt_handle: &str,+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;++ /// Delay redelivery of a queue receipt by changing its visibility period.+ fn delay_message(+ &self,+ receipt_handle: &str,+ delay: Duration,+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;+}++/// Repository for idempotent webhook deliveries and atomic attempt transitions.+pub trait WebhookDeliveryRepository: Clone + Send + Sync + 'static {+ /// Repository error.+ type Err: Into<anyhow::Error> + Send;++ /// Create a delivery if needed and load its current state and webhook configuration.+ ///+ /// Implementations must enforce idempotency for the message's webhook and event ids.+ /// `None` indicates that the webhook does not exist and no delivery could be prepared.+ fn prepare_delivery(+ &self,+ message: &WebhookEventQueueMessage,+ ) -> impl Future<Output = Result<Option<PreparedWebhookDelivery>, Self::Err>> + Send;++ /// Mark a prepared, non-terminal delivery as canceled without creating an attempt.+ fn cancel_delivery(+ &self,+ delivery_id: &str,+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;++ /// Lock a delivery row and begin its next one-based attempt.+ ///+ /// Implementations must re-check status and `next_attempt_at` while holding the+ /// row lock. `None` means the delivery is terminal or is not ready for an attempt.+ fn begin_attempt(+ &self,+ delivery_id: &str,+ ) -> impl Future<Output = Result<Option<WebhookDeliveryAttempt>, Self::Err>> + Send;++ /// Atomically record a successful attempt and delivered webhook state.+ fn record_success(+ &self,+ attempt: &WebhookDeliveryAttempt,+ details: &WebhookHttpOutcomeDetails,+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;++ /// Atomically record a retryable failure and schedule the next attempt.+ fn record_retryable_failure(+ &self,+ attempt: &WebhookDeliveryAttempt,+ details: &WebhookHttpOutcomeDetails,+ next_attempt_at: DateTime<Utc>,+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;++ /// Atomically record an attempt and delivery as permanently failed.+ fn record_permanent_failure(+ &self,+ attempt: &WebhookDeliveryAttempt,+ details: &WebhookHttpOutcomeDetails,+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;++ /// Atomically record a failed attempt and delivery as exhausted.+ fn record_exhaustion(+ &self,+ attempt: &WebhookDeliveryAttempt,+ details: &WebhookHttpOutcomeDetails,+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;+}++/// Client that sends one signed webhook request using current webhook configuration.+pub trait WebhookDeliveryClient: Clone + Send + Sync + 'static {+ /// Client error for failures that cannot be represented as an HTTP outcome.+ type Err: Into<anyhow::Error> + Send;++ /// Deliver the complete broker envelope to a webhook endpoint.+ fn deliver(+ &self,+ webhook: &Webhook,+ event: &NormalizedWebhookEvent,+ ) -> impl Future<Output = Result<WebhookHttpOutcome, Self::Err>> + Send;+}++/// Domain service that prepares, attempts, and records one queued webhook event.+pub trait WebhookEventDeliveryService: Clone + Send + Sync + 'static {+ /// Delivery service error.+ type Err: Into<anyhow::Error> + Send;++ /// Process one valid queue payload and choose how its receipt should be handled.+ fn deliver_event(+ &self,+ message: WebhookEventQueueMessage,+ ) -> impl Future<Output = Result<WebhookWorkerDisposition, Self::Err>> + Send;+}+/// Client used to send a signed webhook validation delivery.pub trait WebhookValidationClient: Clone + Send + Sync + 'static {/// Client error.diff --git a/rust/cloud-storage/webhook/src/domain/service.rs b/rust/cloud-storage/webhook/src/domain/service.rsindex b90f9ccfd4..74a6fabfe8 100644--- a/rust/cloud-storage/webhook/src/domain/service.rs+++ b/rust/cloud-storage/webhook/src/domain/service.rs@@ -3,7 +3,8 @@use super::{models::{CreateWebhookRequest, PatchWebhookRequest, ValidateWebhookResponse, Webhook,- WebhookFilters, WebhookId, WebhookScope, WebhookValidationResult,+ WebhookEndpointSchemePolicy, WebhookFilters, WebhookId, WebhookScope,+ WebhookValidationResult,},ports::{WebhookError, WebhookRepo, WebhookService, WebhookValidationClient},};@@ -24,14 +25,29 @@ const VALIDATION_EVENT_NAME: &str = "webhook.validation.test";pub struct WebhookServiceImpl<R, V> {repo: R,validation_client: V,+ endpoint_scheme_policy: WebhookEndpointSchemePolicy,}impl<R, V> WebhookServiceImpl<R, V> {- /// Create a webhook service.+ /// Create a webhook service that requires public HTTPS endpoints.pub fn new(repo: R, validation_client: V) -> Self {+ Self::new_with_endpoint_scheme_policy(+ repo,+ validation_client,+ WebhookEndpointSchemePolicy::HttpsOnly,+ )+ }++ /// Create a webhook service with an explicit endpoint policy.+ pub fn new_with_endpoint_scheme_policy(+ repo: R,+ validation_client: V,+ endpoint_scheme_policy: WebhookEndpointSchemePolicy,+ ) -> Self {Self {repo,validation_client,+ endpoint_scheme_policy,}}}@@ -46,11 +62,14 @@ fn validate_name(name: &str) -> Result<(), WebhookError> {Ok(())}-fn validate_endpoint_url(endpoint_url: &str) -> Result<(), WebhookError> {+fn validate_endpoint_url(+ endpoint_url: &str,+ endpoint_scheme_policy: WebhookEndpointSchemePolicy,+) -> Result<(), WebhookError> {let url = Url::parse(endpoint_url).map_err(|_| WebhookError::BadRequest("endpoint_url must be a valid URL".to_string()))?;- if url.scheme() != "https" {+ if !endpoint_scheme_policy.allows(url.scheme()) {return Err(WebhookError::BadRequest("endpoint_url must use https".to_string(),));@@ -62,7 +81,7 @@ fn validate_endpoint_url(endpoint_url: &str) -> Result<(), WebhookError> {));};- if is_blocked_endpoint_host(host) {+ if !endpoint_scheme_policy.allows_local_addresses() && is_blocked_endpoint_host(host) {return Err(WebhookError::BadRequest("endpoint_url host is not allowed".to_string(),));@@ -171,18 +190,24 @@ fn validate_filters(filters: &WebhookFilters) -> Result<(), WebhookError> {Ok(())}-fn validate_create_request(request: &CreateWebhookRequest) -> Result<(), WebhookError> {+fn validate_create_request(+ request: &CreateWebhookRequest,+ endpoint_scheme_policy: WebhookEndpointSchemePolicy,+) -> Result<(), WebhookError> {validate_name(&request.name)?;- validate_endpoint_url(&request.endpoint_url)?;+ validate_endpoint_url(&request.endpoint_url, endpoint_scheme_policy)?;validate_filters(&request.filters)}-fn validate_patch_request(request: &PatchWebhookRequest) -> Result<(), WebhookError> {+fn validate_patch_request(+ request: &PatchWebhookRequest,+ endpoint_scheme_policy: WebhookEndpointSchemePolicy,+) -> Result<(), WebhookError> {if let Some(name) = &request.name {validate_name(name)?;}if let Some(endpoint_url) = &request.endpoint_url {- validate_endpoint_url(endpoint_url)?;+ validate_endpoint_url(endpoint_url, endpoint_scheme_policy)?;}if let Some(filters) = &request.filters {validate_filters(filters)?;@@ -291,7 +316,7 @@ wherecaller: MacroUserIdStr<'static>,request: CreateWebhookRequest,) -> Result<Webhook, WebhookError> {- validate_create_request(&request)?;+ validate_create_request(&request, self.endpoint_scheme_policy)?;let workspace_id = self.workspace_id_for_scope(caller.clone(), request.scope)@@ -318,7 +343,7 @@ wherelet webhook = self.load_authorized_webhook(caller, webhook_id.clone()).await?;- validate_patch_request(&request)?;+ validate_patch_request(&request, self.endpoint_scheme_policy)?;let reset_validity = request.endpoint_url.is_some() || request.headers.is_some();let patched = selfdiff --git a/rust/cloud-storage/webhook/src/domain/service_test.rs b/rust/cloud-storage/webhook/src/domain/service_test.rsindex 52572e7bb4..617c1f22fc 100644--- a/rust/cloud-storage/webhook/src/domain/service_test.rs+++ b/rust/cloud-storage/webhook/src/domain/service_test.rs@@ -1,7 +1,8 @@use super::{models::{- CreateWebhookRequest, PatchWebhookRequest, ValidateWebhookResponse, Webhook, WebhookFilter,- WebhookFilters, WebhookScope, WebhookStatus, WebhookValidationResult,+ CreateWebhookRequest, PatchWebhookRequest, ValidateWebhookResponse, Webhook,+ WebhookEndpointSchemePolicy, WebhookFilter, WebhookFilters, WebhookScope, WebhookStatus,+ WebhookValidationResult,},ports::{WebhookError, WebhookRepo, WebhookService, WebhookValidationClient},service::WebhookServiceImpl,@@ -352,6 +353,33 @@ async fn invalid_http_endpoint_is_rejected() {assert_bad_request(service.create_webhook(caller(), request).await);}+#[tokio::test]+async fn local_addresses_are_allowed_when_policy_permits_them() {+ let service = WebhookServiceImpl::new_with_endpoint_scheme_policy(+ FakeRepo::default(),+ FakeValidationClient::default(),+ WebhookEndpointSchemePolicy::HttpAndHttps,+ );++ for endpoint_url in [+ "http://localhost/webhook",+ "http://127.0.0.1/webhook",+ "http://10.1.2.3/webhook",+ "http://[::1]/webhook",+ "http://[fe80::1]/webhook",+ ] {+ let mut request = create_request();+ request.endpoint_url = endpoint_url.to_string();++ let webhook = service+ .create_webhook(caller(), request)+ .await+ .expect("local address should be allowed");++ assert_eq!(webhook.endpoint_url, endpoint_url);+ }+}+#[tokio::test]async fn private_and_link_local_endpoints_are_rejected() {let repo = FakeRepo::default();diff --git a/rust/cloud-storage/webhook/src/inbound/kafka_consumer.rs b/rust/cloud-storage/webhook/src/inbound/kafka_consumer.rsindex 58491055d5..c60eb65370 100644--- a/rust/cloud-storage/webhook/src/inbound/kafka_consumer.rs+++ b/rust/cloud-storage/webhook/src/inbound/kafka_consumer.rs@@ -287,7 +287,6 @@ wherematch WebhookConsumerEvent::decode(message.topic(), payload) {Ok(event) => {tracing::trace!(- ?event,partition = message.partition(),offset = message.offset(),"decoded broker event"diff --git a/rust/cloud-storage/webhook/src/inbound/kafka_consumer/test.rs b/rust/cloud-storage/webhook/src/inbound/kafka_consumer/test.rsindex e8d129f76b..acc0274c64 100644--- a/rust/cloud-storage/webhook/src/inbound/kafka_consumer/test.rs+++ b/rust/cloud-storage/webhook/src/inbound/kafka_consumer/test.rs@@ -72,11 +72,9 @@ struct FlakyIngestionService {impl FlakyIngestionService {fn failure(&self) -> WebhookEventIngestionError {if self.transient {- WebhookEventIngestionError::EntityAccess(AccessError::DatabaseError(- sqlx::Error::PoolTimedOut,- ))+ WebhookEventIngestionError::Enqueue(anyhow::anyhow!("queue unavailable"))} else {- WebhookEventIngestionError::Internal(anyhow::anyhow!("permanent failure"))+ WebhookEventIngestionError::EntityAccess(AccessError::Unauthorized)}}}diff --git a/rust/cloud-storage/webhook/src/inbound/mod.rs b/rust/cloud-storage/webhook/src/inbound/mod.rsindex 2b84ac5b7d..6efff1c630 100644--- a/rust/cloud-storage/webhook/src/inbound/mod.rs+++ b/rust/cloud-storage/webhook/src/inbound/mod.rs@@ -1,4 +1,4 @@-//! Inbound adapters: HTTP handlers and the Kafka event consumer.+//! Inbound adapters: HTTP handlers, the Kafka event consumer, and queue worker.#[cfg(feature = "inbound")]pub mod axum_router;@@ -8,3 +8,6 @@ mod axum_router_test;#[cfg(feature = "consumer")]pub mod kafka_consumer;++#[cfg(feature = "worker")]+pub mod worker;diff --git a/rust/cloud-storage/webhook/src/inbound/worker.rs b/rust/cloud-storage/webhook/src/inbound/worker.rsnew file mode 100644index 0000000000..5f1bd94291--- /dev/null+++ b/rust/cloud-storage/webhook/src/inbound/worker.rs@@ -0,0 +1,152 @@+//! Inbound worker for queued webhook events.+//!+//! The worker owns queue polling and receipt handling. Delivery state, HTTP+//! attempts, and retry policy remain in the domain delivery service.++#[cfg(test)]+mod test;++use std::time::Duration;++use crate::domain::{+ models::{RawWebhookEventQueueMessage, WebhookEventQueueMessage, WebhookWorkerDisposition},+ ports::{WebhookEventDeliveryService, WebhookEventQueue},+};++const POLL_RETRY_DELAY: Duration = Duration::from_secs(1);++/// Continuously receives and processes queued webhook events.+///+/// Messages in each received batch are processed sequentially so events from+/// the same FIFO message group cannot be reordered within a batch.+pub struct WebhookEventWorker<Q, S> {+ queue: Q,+ service: S,+}++impl<Q, S> WebhookEventWorker<Q, S>+where+ Q: WebhookEventQueue,+ S: WebhookEventDeliveryService,+{+ /// Create a worker using the queue transport and delivery domain service.+ pub fn new(queue: Q, service: S) -> Self {+ Self { queue, service }+ }++ /// Run the webhook event worker until the task is canceled.+ pub async fn run(&self) -> ! {+ loop {+ self.poll_and_process_batch().await;+ }+ }++ async fn poll_and_process_batch(&self) {+ let messages = match self.queue.receive_messages().await {+ Ok(messages) => messages,+ Err(error) => {+ let error: anyhow::Error = error.into();+ tracing::error!(error = ?error, "failed to receive webhook event messages");+ tokio::time::sleep(POLL_RETRY_DELAY).await;+ return;+ }+ };++ if messages.is_empty() {+ tokio::time::sleep(POLL_RETRY_DELAY).await;+ return;+ }++ tracing::debug!(count = messages.len(), "received webhook event messages");+ for message in messages {+ self.process_message(message).await;+ }+ }++ #[tracing::instrument(+ skip(self, message),+ fields(message_id = message.message_id.as_deref().unwrap_or("unknown"))+ )]+ async fn process_message(&self, message: RawWebhookEventQueueMessage) {+ let Some(body) = message.body.as_deref() else {+ tracing::warn!("acknowledging webhook event message without a body");+ self.acknowledge_poison_message(message.receipt_handle.as_deref())+ .await;+ return;+ };++ let queue_message = match serde_json::from_str::<WebhookEventQueueMessage>(body) {+ Ok(queue_message) => queue_message,+ Err(error) => {+ tracing::warn!(+ error = ?error,+ "acknowledging malformed webhook event message"+ );+ self.acknowledge_poison_message(message.receipt_handle.as_deref())+ .await;+ return;+ }+ };++ if !queue_message.has_supported_version() {+ tracing::warn!(+ version = queue_message.version,+ "acknowledging unsupported webhook event message version"+ );+ self.acknowledge_poison_message(message.receipt_handle.as_deref())+ .await;+ return;+ }++ let Some(receipt_handle) = message.receipt_handle.as_deref() else {+ tracing::error!(+ webhook_id = %queue_message.webhook_id,+ event_id = %queue_message.event.event_id,+ "cannot process webhook event message without a receipt handle"+ );+ return;+ };++ let disposition = match self.service.deliver_event(queue_message).await {+ Ok(disposition) => disposition,+ Err(error) => {+ let error: anyhow::Error = error.into();+ tracing::error!(error = ?error, "failed to process webhook event message");+ return;+ }+ };++ match disposition {+ WebhookWorkerDisposition::Acknowledge => {+ if let Err(error) = self.queue.delete_message(receipt_handle).await {+ let error: anyhow::Error = error.into();+ tracing::error!(error = ?error, "failed to acknowledge webhook event message");+ }+ }+ WebhookWorkerDisposition::RetryAfter(delay) => {+ if let Err(error) = self.queue.delay_message(receipt_handle, delay).await {+ let error: anyhow::Error = error.into();+ tracing::error!(+ error = ?error,+ delay_seconds = delay.as_secs(),+ "failed to update webhook event message visibility"+ );+ }+ }+ }+ }++ async fn acknowledge_poison_message(&self, receipt_handle: Option<&str>) {+ let Some(receipt_handle) = receipt_handle else {+ tracing::error!(+ "cannot acknowledge poison webhook event message without a receipt handle"+ );+ return;+ };++ if let Err(error) = self.queue.delete_message(receipt_handle).await {+ let error: anyhow::Error = error.into();+ tracing::error!(error = ?error, "failed to acknowledge poison webhook event message");+ }+ }+}diff --git a/rust/cloud-storage/webhook/src/inbound/worker/test.rs b/rust/cloud-storage/webhook/src/inbound/worker/test.rsnew file mode 100644index 0000000000..3974164b6d--- /dev/null+++ b/rust/cloud-storage/webhook/src/inbound/worker/test.rs@@ -0,0 +1,391 @@+use super::*;+use crate::domain::models::{NormalizedWebhookEvent, WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION};+use chrono::{DateTime, Utc};+use std::{+ collections::{HashSet, VecDeque},+ fmt,+ sync::{Arc, Mutex},+};+use tokio::time::Instant;++#[derive(Debug, Clone, Copy)]+struct TestError(&'static str);++impl fmt::Display for TestError {+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {+ formatter.write_str(self.0)+ }+}++impl std::error::Error for TestError {}++#[derive(Default)]+struct QueueState {+ receive_results: VecDeque<Result<Vec<RawWebhookEventQueueMessage>, TestError>>,+ receive_count: usize,+ delete_requests: Vec<String>,+ delay_requests: Vec<(String, Duration)>,+ failed_deletes: HashSet<String>,+ failed_delays: HashSet<String>,+ operations: Option<Arc<Mutex<Vec<String>>>>,+}++#[derive(Clone, Default)]+struct TestQueue {+ state: Arc<Mutex<QueueState>>,+}++impl TestQueue {+ fn with_receive_result(result: Result<Vec<RawWebhookEventQueueMessage>, TestError>) -> Self {+ let queue = Self::default();+ queue+ .state+ .lock()+ .expect("queue state lock")+ .receive_results+ .push_back(result);+ queue+ }++ fn fail_delete(&self, receipt_handle: &str) {+ self.state+ .lock()+ .expect("queue state lock")+ .failed_deletes+ .insert(receipt_handle.to_string());+ }++ fn fail_delay(&self, receipt_handle: &str) {+ self.state+ .lock()+ .expect("queue state lock")+ .failed_delays+ .insert(receipt_handle.to_string());+ }++ fn set_operations(&self, operations: Arc<Mutex<Vec<String>>>) {+ self.state.lock().expect("queue state lock").operations = Some(operations);+ }++ fn receive_count(&self) -> usize {+ self.state.lock().expect("queue state lock").receive_count+ }++ fn delete_requests(&self) -> Vec<String> {+ self.state+ .lock()+ .expect("queue state lock")+ .delete_requests+ .clone()+ }++ fn delay_requests(&self) -> Vec<(String, Duration)> {+ self.state+ .lock()+ .expect("queue state lock")+ .delay_requests+ .clone()+ }+}++impl WebhookEventQueue for TestQueue {+ type Err = TestError;++ async fn receive_messages(&self) -> Result<Vec<RawWebhookEventQueueMessage>, Self::Err> {+ let mut state = self.state.lock().expect("queue state lock");+ state.receive_count += 1;+ state.receive_results.pop_front().unwrap_or(Ok(Vec::new()))+ }++ async fn delete_message(&self, receipt_handle: &str) -> Result<(), Self::Err> {+ let mut state = self.state.lock().expect("queue state lock");+ state.delete_requests.push(receipt_handle.to_string());+ if let Some(operations) = &state.operations {+ operations+ .lock()+ .expect("operations lock")+ .push(format!("delete:{receipt_handle}"));+ }+ if state.failed_deletes.contains(receipt_handle) {+ return Err(TestError("delete failed"));+ }+ Ok(())+ }++ async fn delay_message(&self, receipt_handle: &str, delay: Duration) -> Result<(), Self::Err> {+ let mut state = self.state.lock().expect("queue state lock");+ state+ .delay_requests+ .push((receipt_handle.to_string(), delay));+ if state.failed_delays.contains(receipt_handle) {+ return Err(TestError("visibility update failed"));+ }+ Ok(())+ }+}++struct ServiceState {+ outcomes: VecDeque<Result<WebhookWorkerDisposition, TestError>>,+ messages: Vec<WebhookEventQueueMessage>,+ operations: Option<Arc<Mutex<Vec<String>>>>,+}++#[derive(Clone)]+struct TestDeliveryService {+ state: Arc<Mutex<ServiceState>>,+}++impl TestDeliveryService {+ fn new(+ outcomes: impl IntoIterator<Item = Result<WebhookWorkerDisposition, TestError>>,+ ) -> Self {+ Self {+ state: Arc::new(Mutex::new(ServiceState {+ outcomes: outcomes.into_iter().collect(),+ messages: Vec::new(),+ operations: None,+ })),+ }+ }++ fn set_operations(&self, operations: Arc<Mutex<Vec<String>>>) {+ self.state.lock().expect("service state lock").operations = Some(operations);+ }++ fn messages(&self) -> Vec<WebhookEventQueueMessage> {+ self.state+ .lock()+ .expect("service state lock")+ .messages+ .clone()+ }+}++impl WebhookEventDeliveryService for TestDeliveryService {+ type Err = TestError;++ async fn deliver_event(+ &self,+ message: WebhookEventQueueMessage,+ ) -> Result<WebhookWorkerDisposition, Self::Err> {+ let mut state = self.state.lock().expect("service state lock");+ if let Some(operations) = &state.operations {+ operations+ .lock()+ .expect("operations lock")+ .push(format!("service:{}", message.event.event_id));+ }+ state.messages.push(message);+ state+ .outcomes+ .pop_front()+ .expect("a delivery outcome for every valid test message")+ }+}++fn queue_message(event_id: &str) -> WebhookEventQueueMessage {+ WebhookEventQueueMessage::new(+ "wh_test".to_string(),+ NormalizedWebhookEvent {+ event_id: event_id.to_string(),+ schema_version: 1,+ event_name: "document.updated".to_string(),+ entity_type: "document".to_string(),+ entity_id: "doc_test".to_string(),+ ordering_key: "doc_test".to_string(),+ occurred_at: "2026-07-10T00:00:00Z"+ .parse::<DateTime<Utc>>()+ .expect("valid timestamp"),+ broker_envelope: serde_json::json!({"event_id": event_id}),+ },+ )+}++fn raw_message(+ message_id: &str,+ body: Option<String>,+ receipt_handle: Option<&str>,+) -> RawWebhookEventQueueMessage {+ RawWebhookEventQueueMessage {+ message_id: Some(message_id.to_string()),+ body,+ receipt_handle: receipt_handle.map(str::to_string),+ }+}++fn valid_raw_message(event_id: &str, receipt_handle: &str) -> RawWebhookEventQueueMessage {+ raw_message(+ event_id,+ Some(serde_json::to_string(&queue_message(event_id)).expect("serializable queue message")),+ Some(receipt_handle),+ )+}++#[tokio::test(start_paused = true)]+async fn empty_poll_sleeps_before_polling_again() {+ let queue = TestQueue::with_receive_result(Ok(Vec::new()));+ let worker = WebhookEventWorker::new(queue.clone(), TestDeliveryService::new([]));+ let started_at = Instant::now();++ worker.poll_and_process_batch().await;++ assert_eq!(Instant::now().duration_since(started_at), POLL_RETRY_DELAY);+ assert_eq!(queue.receive_count(), 1);+}++#[tokio::test(start_paused = true)]+async fn failed_poll_sleeps_before_polling_again() {+ let queue = TestQueue::with_receive_result(Err(TestError("receive failed")));+ let worker = WebhookEventWorker::new(queue.clone(), TestDeliveryService::new([]));+ let started_at = Instant::now();++ worker.poll_and_process_batch().await;++ assert_eq!(Instant::now().duration_since(started_at), POLL_RETRY_DELAY);+ assert_eq!(queue.receive_count(), 1);+}++#[tokio::test]+async fn acknowledges_missing_malformed_and_unsupported_messages() {+ let mut unsupported = queue_message("unsupported");+ unsupported.version = WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION + 1;+ let messages = vec![+ raw_message("missing", None, Some("receipt-missing")),+ raw_message(+ "malformed",+ Some("not valid json".to_string()),+ Some("receipt-malformed"),+ ),+ raw_message(+ "unsupported",+ Some(serde_json::to_string(&unsupported).expect("serializable queue message")),+ Some("receipt-unsupported"),+ ),+ ];+ let queue = TestQueue::with_receive_result(Ok(messages));+ let service = TestDeliveryService::new([]);+ let worker = WebhookEventWorker::new(queue.clone(), service.clone());++ worker.poll_and_process_batch().await;++ assert_eq!(+ queue.delete_requests(),+ [+ "receipt-missing",+ "receipt-malformed",+ "receipt-unsupported"+ ]+ );+ assert!(service.messages().is_empty());+}++#[tokio::test]+async fn deletes_valid_messages_after_acknowledge_disposition() {+ let message = queue_message("event-1");+ let queue = TestQueue::with_receive_result(Ok(vec![valid_raw_message(+ &message.event.event_id,+ "receipt-1",+ )]));+ let service = TestDeliveryService::new([Ok(WebhookWorkerDisposition::Acknowledge)]);+ let worker = WebhookEventWorker::new(queue.clone(), service.clone());++ worker.poll_and_process_batch().await;++ assert_eq!(service.messages(), [message]);+ assert_eq!(queue.delete_requests(), ["receipt-1"]);+ assert!(queue.delay_requests().is_empty());+}++#[tokio::test]+async fn changes_visibility_after_retry_disposition() {+ let delay = Duration::from_secs(30);+ let queue = TestQueue::with_receive_result(Ok(vec![valid_raw_message("event-1", "receipt-1")]));+ let service = TestDeliveryService::new([Ok(WebhookWorkerDisposition::RetryAfter(delay))]);+ let worker = WebhookEventWorker::new(queue.clone(), service);++ worker.poll_and_process_batch().await;++ assert_eq!(queue.delay_requests(), [("receipt-1".to_string(), delay)]);+ assert!(queue.delete_requests().is_empty());+}++#[tokio::test]+async fn failed_acknowledgment_leaves_the_message_for_redelivery() {+ let queue = TestQueue::with_receive_result(Ok(vec![valid_raw_message("event-1", "receipt-1")]));+ queue.fail_delete("receipt-1");+ let service = TestDeliveryService::new([Ok(WebhookWorkerDisposition::Acknowledge)]);+ let worker = WebhookEventWorker::new(queue.clone(), service);++ worker.poll_and_process_batch().await;++ assert_eq!(queue.delete_requests(), ["receipt-1"]);+ assert!(queue.delay_requests().is_empty());+}++#[tokio::test]+async fn failed_visibility_update_leaves_the_message_for_redelivery() {+ let delay = Duration::from_secs(60);+ let queue = TestQueue::with_receive_result(Ok(vec![valid_raw_message("event-1", "receipt-1")]));+ queue.fail_delay("receipt-1");+ let service = TestDeliveryService::new([Ok(WebhookWorkerDisposition::RetryAfter(delay))]);+ let worker = WebhookEventWorker::new(queue.clone(), service);++ worker.poll_and_process_batch().await;++ assert_eq!(queue.delay_requests(), [("receipt-1".to_string(), delay)]);+ assert!(queue.delete_requests().is_empty());+}++#[tokio::test]+async fn service_failures_do_not_acknowledge_or_stop_the_batch() {+ let queue = TestQueue::with_receive_result(Ok(vec![+ valid_raw_message("event-1", "receipt-1"),+ valid_raw_message("event-2", "receipt-2"),+ ]));+ let service = TestDeliveryService::new([+ Err(TestError("service failed")),+ Ok(WebhookWorkerDisposition::Acknowledge),+ ]);+ let worker = WebhookEventWorker::new(queue.clone(), service.clone());++ worker.poll_and_process_batch().await;++ assert_eq!(+ service+ .messages()+ .into_iter()+ .map(|message| message.event.event_id)+ .collect::<Vec<_>>(),+ ["event-1", "event-2"]+ );+ assert_eq!(queue.delete_requests(), ["receipt-2"]);+ assert!(queue.delay_requests().is_empty());+}++#[tokio::test]+async fn processes_each_batch_sequentially() {+ let operations = Arc::new(Mutex::new(Vec::new()));+ let queue = TestQueue::with_receive_result(Ok(vec![+ valid_raw_message("event-1", "receipt-1"),+ valid_raw_message("event-2", "receipt-2"),+ ]));+ queue.set_operations(operations.clone());+ let service = TestDeliveryService::new([+ Ok(WebhookWorkerDisposition::Acknowledge),+ Ok(WebhookWorkerDisposition::Acknowledge),+ ]);+ service.set_operations(operations.clone());+ let worker = WebhookEventWorker::new(queue, service);++ worker.poll_and_process_batch().await;++ assert_eq!(+ *operations.lock().expect("operations lock"),+ [+ "service:event-1",+ "delete:receipt-1",+ "service:event-2",+ "delete:receipt-2",+ ]+ );+}diff --git a/rust/cloud-storage/webhook/src/lib.rs b/rust/cloud-storage/webhook/src/lib.rsindex 770873f2fa..b409b36db6 100644--- a/rust/cloud-storage/webhook/src/lib.rs+++ b/rust/cloud-storage/webhook/src/lib.rs@@ -1,43 +1,10 @@#![deny(missing_docs)]-//! Webhook management hex crate.-//!-//! This crate currently contains the core application-layer webhook pieces for-//! the first iteration:-//!-//! - database tables for event ingestion, webhooks, deliveries, and delivery-//! attempts;-//! - a typed `filters` model for matching webhook deliveries, shaped as an-//! array of objects such as-//! `[{"events": ["document.created"], "ids": ["doc_123"]}]`, where-//! `ids` is optional and an absent `ids` field matches all entity ids;-//! - a repository that can create, get, patch, and update webhook validity via-//! `is_valid`; it can also list active webhooks whose typed `filters` match-//! an event with-//! `domain::ports::WebhookRepo::list_active_webhooks_matching_event`;-//! - inbound handlers for `POST /webhooks`, `PATCH /webhooks/{webhook_id}`, and-//! `POST /webhooks/{webhook_id}/validate`;-//! - validation that sends a signed `webhook.validation.test` event and persists-//! the result in `is_valid`;-//! - validation attempt rate limiting with the existing `rate_limit` crate using-//! the key shape `per-user-validate-webhook:{macro_user_id}:{webhook_id}` and-//! a limit of 10 attempts per 3600-second window;-//! - a Kafka consumer (`inbound::kafka_consumer`, feature `consumer`) that-//! subscribes to the `macro.documents` and `macro.channels` topics and hands-//! every event to the `domain::ingestion::WebhookEventIngestionService`-//! (feature `ingestion`), whose per-event handlers are still stubs.-//!-//! The current implementation intentionally excludes delivery workers, SQS/SNS-//! infrastructure, redelivery, and auto-pause behavior.-//!-//! Temporary limitations: IDs use prefixed UUIDv7 strings (`wh_<uuid_v7>`)-//! rather than true ULIDs. Custom headers are stored as JSON in `headers`.-//! Webhook signing secrets are intentionally stored as plaintext in-//! `signing_secret`, matching the existing bot secret storage approach.+#![doc = include_str!("../README.md")]/// Domain models, ports, and services.pub mod domain;-#[cfg(any(feature = "inbound", feature = "consumer"))]-/// HTTP and Kafka consumer adapters.+#[cfg(any(feature = "inbound", feature = "consumer", feature = "worker"))]+/// HTTP, Kafka consumer, and queue worker adapters.pub mod inbound;#[cfg(feature = "outbound")]/// Postgres and HTTP adapters.diff --git a/rust/cloud-storage/webhook/src/outbound.rs b/rust/cloud-storage/webhook/src/outbound.rsindex 6e941a08e0..59044264f5 100644--- a/rust/cloud-storage/webhook/src/outbound.rs+++ b/rust/cloud-storage/webhook/src/outbound.rs@@ -1,7 +1,13 @@//! Outbound adapters.+pub mod http_delivery;pub mod http_validator;+pub mod pg_delivery_repository;pub mod pg_repository;+pub mod sqs_queue;+pub use http_delivery::ReqwestWebhookDeliveryClient;pub use http_validator::ReqwestWebhookValidationClient;+pub use pg_delivery_repository::PgWebhookDeliveryRepository;pub use pg_repository::PgRepository;+pub use sqs_queue::SqsWebhookQueue;diff --git a/rust/cloud-storage/webhook/src/outbound/http_delivery.rs b/rust/cloud-storage/webhook/src/outbound/http_delivery.rsnew file mode 100644index 0000000000..6ec8f78b50--- /dev/null+++ b/rust/cloud-storage/webhook/src/outbound/http_delivery.rs@@ -0,0 +1,298 @@+//! Signed HTTP webhook delivery client.++#[cfg(test)]+#[path = "http_delivery_test.rs"]+mod http_delivery_test;++use super::http_validator::{+ EndpointValidationError, ReqwestWebhookValidationClient, is_reserved_macro_header,+ signature_header, validate_resolved_endpoint_url,+};+use crate::domain::{+ models::{+ NormalizedWebhookEvent, Webhook, WebhookHeaders, WebhookHttpOutcome,+ WebhookHttpOutcomeDetails,+ },+ ports::WebhookDeliveryClient,+};+use reqwest::{+ Client, Request, Response, StatusCode, Url,+ header::{CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue},+};+use std::time::{Duration, Instant};++const EVENT_HEADER: &str = "x-macro-event";+const EVENT_ID_HEADER: &str = "x-macro-event-id";+const TIMESTAMP_HEADER: &str = "x-macro-timestamp";+const SIGNATURE_HEADER: &str = "x-macro-signature";+const REDACTED_HEADER_VALUE: &str = "[REDACTED]";+const RESPONSE_PREVIEW_MAX_BYTES: usize = 4096;++/// Reqwest-backed implementation of [`WebhookDeliveryClient`].+///+/// This is the same cloneable, redirect-disabled client used for webhook+/// validation, so validation and production requests share one security policy.+pub type ReqwestWebhookDeliveryClient = ReqwestWebhookValidationClient;++impl WebhookDeliveryClient for ReqwestWebhookValidationClient {+ type Err = anyhow::Error;++ async fn deliver(+ &self,+ webhook: &Webhook,+ event: &NormalizedWebhookEvent,+ ) -> Result<WebhookHttpOutcome, Self::Err> {+ let started_at = Instant::now();+ let raw_body = serde_json::to_vec(&event.broker_envelope)?;+ let timestamp = chrono::Utc::now().timestamp().to_string();+ let signature = signature_header(&webhook.signing_secret, ×tamp, &raw_body)?;+ let headers = match delivery_headers(webhook, event, ×tamp, &signature) {+ Ok(headers) => headers,+ Err(message) => {+ return Ok(WebhookHttpOutcome::PermanentFailure(+ configuration_failure_details(+ started_at.elapsed(),+ "header_configuration",+ message,+ ),+ ));+ }+ };+ let url = match validate_resolved_endpoint_url(+ &webhook.endpoint_url,+ self.endpoint_scheme_policy,+ )+ .await+ {+ Ok(url) => url,+ Err(error) => return Ok(endpoint_failure(started_at.elapsed(), error)),+ };+ let request = match delivery_request(&self.client, url, headers, raw_body) {+ Ok(request) => request,+ Err(_) => {+ return Ok(WebhookHttpOutcome::PermanentFailure(+ configuration_failure_details(+ started_at.elapsed(),+ "request_configuration",+ "webhook request configuration is invalid",+ ),+ ));+ }+ };++ let response = match self.client.execute(request).await {+ Ok(response) => response,+ Err(error) => {+ let details = transport_failure_details(started_at.elapsed(), &error);+ if error.is_builder() {+ return Ok(WebhookHttpOutcome::PermanentFailure(details));+ }+ return Ok(WebhookHttpOutcome::RetryableFailure(details));+ }+ };++ Ok(response_outcome(response, started_at).await)+ }+}++fn delivery_headers(+ webhook: &Webhook,+ event: &NormalizedWebhookEvent,+ timestamp: &str,+ signature: &str,+) -> Result<HeaderMap, &'static str> {+ if webhook+ .headers+ .keys()+ .any(|name| is_reserved_macro_header(name))+ {+ return Err("reserved Macro headers cannot be overridden");+ }++ let mut headers = HeaderMap::new();+ for (name, value) in &webhook.headers {+ let name = HeaderName::from_bytes(name.as_bytes())+ .map_err(|_| "custom webhook headers are invalid")?;+ let mut value =+ HeaderValue::from_str(value).map_err(|_| "custom webhook headers are invalid")?;+ value.set_sensitive(true);+ headers.insert(name, value);+ }++ headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));+ insert_header(&mut headers, EVENT_HEADER, &event.event_name, false)?;+ insert_header(&mut headers, EVENT_ID_HEADER, &event.event_id, false)?;+ insert_header(&mut headers, TIMESTAMP_HEADER, timestamp, false)?;+ insert_header(&mut headers, SIGNATURE_HEADER, signature, true)?;++ Ok(headers)+}++fn insert_header(+ headers: &mut HeaderMap,+ name: &'static str,+ value: &str,+ sensitive: bool,+) -> Result<(), &'static str> {+ let mut value =+ HeaderValue::from_str(value).map_err(|_| "webhook event headers are invalid")?;+ value.set_sensitive(sensitive);+ headers.insert(HeaderName::from_static(name), value);+ Ok(())+}++fn delivery_request(+ client: &Client,+ url: Url,+ headers: HeaderMap,+ raw_body: Vec<u8>,+) -> Result<Request, reqwest::Error> {+ client.post(url).headers(headers).body(raw_body).build()+}++async fn response_outcome(mut response: Response, started_at: Instant) -> WebhookHttpOutcome {+ let status = response.status();+ let response_headers_redacted = Some(redacted_headers(response.headers()));+ let (response_body_preview, preview_error) = read_response_preview(&mut response).await;+ let duration = started_at.elapsed();++ let (error_kind, error_message) = match preview_error {+ Some(error) if error.is_timeout() => (+ Some("response_timeout".to_string()),+ Some("webhook response preview timed out".to_string()),+ ),+ Some(_) => (+ Some("response_read".to_string()),+ Some("webhook response preview could not be read".to_string()),+ ),+ None if !status.is_success() => (+ Some("http_status".to_string()),+ Some(format!("webhook returned HTTP {}", status.as_u16())),+ ),+ None => (None, None),+ };++ classify_http_status(+ status,+ WebhookHttpOutcomeDetails {+ duration,+ response_status: Some(status.as_u16()),+ response_headers_redacted,+ response_body_preview: Some(response_body_preview),+ error_kind,+ error_message,+ },+ )+}++fn classify_http_status(+ status: StatusCode,+ details: WebhookHttpOutcomeDetails,+) -> WebhookHttpOutcome {+ if status.is_success() {+ return WebhookHttpOutcome::Success(details);+ }++ if status == StatusCode::REQUEST_TIMEOUT+ || status == StatusCode::TOO_MANY_REQUESTS+ || status.is_server_error()+ {+ return WebhookHttpOutcome::RetryableFailure(details);+ }++ WebhookHttpOutcome::PermanentFailure(details)+}++async fn read_response_preview(response: &mut Response) -> (String, Option<reqwest::Error>) {+ let mut preview = Vec::new();+ while preview.len() < RESPONSE_PREVIEW_MAX_BYTES {+ match response.chunk().await {+ Ok(Some(chunk)) => {+ let remaining = RESPONSE_PREVIEW_MAX_BYTES - preview.len();+ preview.extend_from_slice(&chunk[..chunk.len().min(remaining)]);+ }+ Ok(None) => break,+ Err(error) => return (utf8_lossy_preview(&preview), Some(error)),+ }+ }++ (utf8_lossy_preview(&preview), None)+}++fn utf8_lossy_preview(bytes: &[u8]) -> String {+ let bytes = &bytes[..bytes.len().min(RESPONSE_PREVIEW_MAX_BYTES)];+ let mut preview = String::from_utf8_lossy(bytes).into_owned();+ if preview.len() > RESPONSE_PREVIEW_MAX_BYTES {+ let mut end = RESPONSE_PREVIEW_MAX_BYTES;+ while !preview.is_char_boundary(end) {+ end -= 1;+ }+ preview.truncate(end);+ }+ preview+}++fn redacted_headers(headers: &HeaderMap) -> WebhookHeaders {+ headers+ .keys()+ .map(|name| (name.as_str().to_string(), REDACTED_HEADER_VALUE.to_string()))+ .collect()+}++fn endpoint_failure(duration: Duration, error: EndpointValidationError) -> WebhookHttpOutcome {+ let details = configuration_failure_details(+ duration,+ if error.is_retryable() {+ "dns_resolution"+ } else {+ "endpoint_configuration"+ },+ error.message(),+ );++ if error.is_retryable() {+ WebhookHttpOutcome::RetryableFailure(details)+ } else {+ WebhookHttpOutcome::PermanentFailure(details)+ }+}++fn configuration_failure_details(+ duration: Duration,+ error_kind: &str,+ error_message: &str,+) -> WebhookHttpOutcomeDetails {+ WebhookHttpOutcomeDetails {+ duration,+ response_status: None,+ response_headers_redacted: None,+ response_body_preview: None,+ error_kind: Some(error_kind.to_string()),+ error_message: Some(error_message.to_string()),+ }+}++fn transport_failure_details(+ duration: Duration,+ error: &reqwest::Error,+) -> WebhookHttpOutcomeDetails {+ let (error_kind, error_message) = if error.is_timeout() {+ ("timeout", "webhook delivery timed out after 5 seconds")+ } else if error.is_builder() {+ (+ "request_configuration",+ "webhook request configuration is invalid",+ )+ } else {+ ("network", "webhook endpoint could not be reached")+ };++ WebhookHttpOutcomeDetails {+ duration,+ response_status: None,+ response_headers_redacted: None,+ response_body_preview: None,+ error_kind: Some(error_kind.to_string()),+ error_message: Some(error_message.to_string()),+ }+}diff --git a/rust/cloud-storage/webhook/src/outbound/http_delivery_test.rs b/rust/cloud-storage/webhook/src/outbound/http_delivery_test.rsnew file mode 100644index 0000000000..9752e497a4--- /dev/null+++ b/rust/cloud-storage/webhook/src/outbound/http_delivery_test.rs@@ -0,0 +1,254 @@+use super::*;+use crate::domain::{+ models::{WebhookFilter, WebhookStatus},+ ports::WebhookDeliveryClient,+};+use chrono::Utc;+use serde_json::json;+use std::collections::BTreeMap;++fn webhook(endpoint_url: &str, headers: WebhookHeaders) -> Webhook {+ Webhook {+ id: "wh_test".to_string(),+ workspace_id: "workspace_test".to_string(),+ name: "Test webhook".to_string(),+ endpoint_url: endpoint_url.to_string(),+ signing_secret: "signing-secret".to_string(),+ headers,+ status: WebhookStatus::Active,+ is_valid: true,+ created_by_user_id: "macro|[email protected]".to_string(),+ created_at: Utc::now(),+ updated_at: Utc::now(),+ deleted_at: None,+ filters: vec![WebhookFilter {+ events: vec!["document.created".to_string()],+ ids: None,+ }],+ }+}++fn event() -> NormalizedWebhookEvent {+ NormalizedWebhookEvent {+ event_id: "evt_test".to_string(),+ schema_version: 1,+ event_name: "document.created".to_string(),+ entity_type: "document".to_string(),+ entity_id: "doc_test".to_string(),+ ordering_key: "doc_test".to_string(),+ occurred_at: Utc::now(),+ broker_envelope: json!({+ "event_id": "evt_test",+ "schema_version": 1,+ "event_type": "document.created",+ "metadata": {"document_id": "doc_test"},+ }),+ }+}++fn outcome_details() -> WebhookHttpOutcomeDetails {+ WebhookHttpOutcomeDetails {+ duration: Duration::from_millis(10),+ response_status: None,+ response_headers_redacted: None,+ response_body_preview: None,+ error_kind: None,+ error_message: None,+ }+}++#[test]+fn builds_signed_request_with_exact_broker_envelope() {+ let webhook = webhook(+ "https://example.com/webhook",+ BTreeMap::from([("Authorization".to_string(), "Bearer secret".to_string())]),+ );+ let event = event();+ let raw_body = serde_json::to_vec(&event.broker_envelope).unwrap();+ let timestamp = "1783699200";+ let signature = signature_header(&webhook.signing_secret, timestamp, &raw_body).unwrap();+ let headers = delivery_headers(&webhook, &event, timestamp, &signature).unwrap();+ let request = delivery_request(+ &Client::new(),+ Url::parse(&webhook.endpoint_url).unwrap(),+ headers,+ raw_body.clone(),+ )+ .unwrap();++ assert_eq!(+ request.body().and_then(reqwest::Body::as_bytes),+ Some(raw_body.as_slice())+ );+ assert_eq!(request.headers()[CONTENT_TYPE], "application/json");+ assert_eq!(request.headers()[EVENT_HEADER], event.event_name);+ assert_eq!(request.headers()[EVENT_ID_HEADER], event.event_id);+ assert_eq!(request.headers()[TIMESTAMP_HEADER], timestamp);+ assert_eq!(request.headers()[SIGNATURE_HEADER], signature);+ assert_eq!(request.headers()["authorization"], "Bearer secret");+ assert!(request.headers()[SIGNATURE_HEADER].is_sensitive());+ assert!(request.headers()["authorization"].is_sensitive());+}++#[test]+fn rejects_reserved_headers_case_insensitively() {+ for name in [+ "X-Macro-Event",+ "x-macro-event-id",+ "X-MACRO-TIMESTAMP",+ "x-Macro-Signature",+ ] {+ let webhook = webhook(+ "https://example.com/webhook",+ BTreeMap::from([(name.to_string(), "override".to_string())]),+ );++ assert_eq!(+ delivery_headers(&webhook, &event(), "123", "v1=signature"),+ Err("reserved Macro headers cannot be overridden")+ );+ }+}++#[test]+fn rejects_invalid_custom_header_names_and_values() {+ let invalid_name = webhook(+ "https://example.com/webhook",+ BTreeMap::from([("bad header".to_string(), "value".to_string())]),+ );+ let invalid_value = webhook(+ "https://example.com/webhook",+ BTreeMap::from([("X-Custom".to_string(), "first\nsecond".to_string())]),+ );++ assert_eq!(+ delivery_headers(&invalid_name, &event(), "123", "v1=signature"),+ Err("custom webhook headers are invalid")+ );+ assert_eq!(+ delivery_headers(&invalid_value, &event(), "123", "v1=signature"),+ Err("custom webhook headers are invalid")+ );+}++#[test]+fn always_uses_json_content_type() {+ let webhook = webhook(+ "https://example.com/webhook",+ BTreeMap::from([("content-type".to_string(), "text/plain".to_string())]),+ );++ let headers = delivery_headers(&webhook, &event(), "123", "v1=signature").unwrap();++ assert_eq!(headers[CONTENT_TYPE], "application/json");+}++#[test]+fn classifies_http_response_statuses() {+ for status in [200, 201, 204, 299] {+ assert!(matches!(+ classify_http_status(StatusCode::from_u16(status).unwrap(), outcome_details()),+ WebhookHttpOutcome::Success(_)+ ));+ }++ for status in [408, 429, 500, 503, 599] {+ assert!(matches!(+ classify_http_status(StatusCode::from_u16(status).unwrap(), outcome_details()),+ WebhookHttpOutcome::RetryableFailure(_)+ ));+ }++ for status in [300, 400, 401, 404, 409, 422] {+ assert!(matches!(+ classify_http_status(StatusCode::from_u16(status).unwrap(), outcome_details()),+ WebhookHttpOutcome::PermanentFailure(_)+ ));+ }+}++#[test]+fn redacts_response_header_values() {+ let mut headers = HeaderMap::new();+ headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));+ headers.insert(+ HeaderName::from_static("x-request-id"),+ HeaderValue::from_static("private-request-id"),+ );++ assert_eq!(+ redacted_headers(&headers),+ BTreeMap::from([+ (+ "content-type".to_string(),+ REDACTED_HEADER_VALUE.to_string()+ ),+ (+ "x-request-id".to_string(),+ REDACTED_HEADER_VALUE.to_string()+ ),+ ])+ );+}++#[test]+fn caps_utf8_lossy_response_preview_at_4096_bytes() {+ let oversized = vec![b'a'; RESPONSE_PREVIEW_MAX_BYTES + 100];+ let invalid_utf8 = vec![0xff; RESPONSE_PREVIEW_MAX_BYTES];++ let preview = utf8_lossy_preview(&oversized);+ let lossy_preview = utf8_lossy_preview(&invalid_utf8);++ assert_eq!(preview.len(), RESPONSE_PREVIEW_MAX_BYTES);+ assert!(lossy_preview.contains('\u{fffd}'));+ assert!(lossy_preview.len() <= RESPONSE_PREVIEW_MAX_BYTES);+}++#[tokio::test]+async fn invalid_endpoint_configuration_is_permanent() {+ let client = ReqwestWebhookDeliveryClient::new().unwrap();+ let outcome = client+ .deliver(+ &webhook("http://example.com/webhook", BTreeMap::new()),+ &event(),+ )+ .await+ .unwrap();++ let WebhookHttpOutcome::PermanentFailure(details) = outcome else {+ panic!("invalid endpoint should be permanent");+ };+ assert_eq!(details.response_status, None);+ assert_eq!(+ details.error_kind.as_deref(),+ Some("endpoint_configuration")+ );+ assert_eq!(+ details.error_message.as_deref(),+ Some("webhook endpoint URL must use HTTPS")+ );+}++#[tokio::test]+async fn reserved_header_configuration_is_permanent_without_dns_lookup() {+ let client = ReqwestWebhookDeliveryClient::new().unwrap();+ let outcome = client+ .deliver(+ &webhook(+ "https://unresolvable.invalid/webhook",+ BTreeMap::from([("X-Macro-Signature".to_string(), "private".to_string())]),+ ),+ &event(),+ )+ .await+ .unwrap();++ let WebhookHttpOutcome::PermanentFailure(details) = outcome else {+ panic!("reserved header should be permanent");+ };+ assert_eq!(details.error_kind.as_deref(), Some("header_configuration"));+ assert_eq!(+ details.error_message.as_deref(),+ Some("reserved Macro headers cannot be overridden")+ );+}diff --git a/rust/cloud-storage/webhook/src/outbound/http_validator.rs b/rust/cloud-storage/webhook/src/outbound/http_validator.rsindex 592387ee4f..2d78309b0a 100644--- a/rust/cloud-storage/webhook/src/outbound/http_validator.rs+++ b/rust/cloud-storage/webhook/src/outbound/http_validator.rs@@ -5,7 +5,7 @@mod http_validator_test;use crate::domain::{- models::{Webhook, WebhookValidationResult},+ models::{Webhook, WebhookEndpointSchemePolicy, WebhookValidationResult},ports::WebhookValidationClient,};use hmac::{Hmac, Mac};@@ -16,25 +16,36 @@ use std::{net::IpAddr, time::Duration};use tokio::net::lookup_host;const EVENT_NAME: &str = "webhook.validation.test";-const REQUEST_TIMEOUT: Duration = Duration::from_secs(5);+pub(super) const REQUEST_TIMEOUT: Duration = Duration::from_secs(5);type HmacSha256 = Hmac<Sha256>;/// Reqwest-backed implementation of [`WebhookValidationClient`].#[derive(Clone)]pub struct ReqwestWebhookValidationClient {- client: Client,+ pub(super) client: Client,+ pub(super) endpoint_scheme_policy: WebhookEndpointSchemePolicy,}impl ReqwestWebhookValidationClient {- /// Create a validation client with the required webhook delivery limits.+ /// Create a validation client that requires public HTTPS endpoints.pub fn new() -> Result<Self, reqwest::Error> {+ Self::new_with_endpoint_scheme_policy(WebhookEndpointSchemePolicy::HttpsOnly)+ }++ /// Create a validation client with an explicit endpoint policy.+ pub fn new_with_endpoint_scheme_policy(+ endpoint_scheme_policy: WebhookEndpointSchemePolicy,+ ) -> Result<Self, reqwest::Error> {let client = Client::builder().timeout(REQUEST_TIMEOUT).redirect(Policy::none()).build()?;- Ok(Self { client })+ Ok(Self {+ client,+ endpoint_scheme_policy,+ })}}@@ -45,9 +56,14 @@ impl WebhookValidationClient for ReqwestWebhookValidationClient {&self,webhook: Webhook,) -> Result<WebhookValidationResult, Self::Err> {- let url = match validate_resolved_endpoint_url(&webhook.endpoint_url).await {+ let url = match validate_resolved_endpoint_url(+ &webhook.endpoint_url,+ self.endpoint_scheme_policy,+ )+ .await+ {Ok(url) => url,- Err(message) => return Ok(failure(None, message)),+ Err(error) => return Ok(failure(None, error.message())),};let event_id = new_validation_event_id();@@ -107,7 +123,7 @@ fn validation_body(webhook_id: &str, event_id: &str) -> Result<Vec<u8>, serde_js}))}-fn signature_header(+pub(super) fn signature_header(secret: &str,timestamp: &str,raw_body: &[u8],@@ -119,45 +135,96 @@ fn signature_header(Ok(format!("v1={}", hex::encode(mac.finalize().into_bytes())))}-async fn validate_resolved_endpoint_url(value: &str) -> Result<Url, &'static str> {- let url = validate_endpoint_url(value)?;+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(super) enum EndpointValidationError {+ InvalidConfiguration(&'static str),+ ResolutionFailure(&'static str),+}++impl EndpointValidationError {+ pub(super) const fn message(self) -> &'static str {+ match self {+ Self::InvalidConfiguration(message) | Self::ResolutionFailure(message) => message,+ }+ }++ pub(super) const fn is_retryable(self) -> bool {+ matches!(self, Self::ResolutionFailure(_))+ }+}++pub(super) async fn validate_resolved_endpoint_url(+ value: &str,+ endpoint_scheme_policy: WebhookEndpointSchemePolicy,+) -> Result<Url, EndpointValidationError> {+ let url = validate_endpoint_url(value, endpoint_scheme_policy)?;let Some(host) = url.host_str().map(str::to_owned) else {- return Err("webhook endpoint URL host is invalid");+ return Err(EndpointValidationError::InvalidConfiguration(+ "webhook endpoint URL host is invalid",+ ));};let Some(port) = url.port_or_known_default() else {- return Err("webhook endpoint URL port is invalid");+ return Err(EndpointValidationError::InvalidConfiguration(+ "webhook endpoint URL port is invalid",+ ));};- let mut resolved = lookup_host((host.as_str(), port))- .await- .map_err(|_| "webhook endpoint host could not be resolved")?;+ let mut resolved =+ match tokio::time::timeout(REQUEST_TIMEOUT, lookup_host((host.as_str(), port))).await {+ Ok(Ok(resolved)) => resolved,+ Ok(Err(_)) => {+ return Err(EndpointValidationError::ResolutionFailure(+ "webhook endpoint host could not be resolved",+ ));+ }+ Err(_) => {+ return Err(EndpointValidationError::ResolutionFailure(+ "webhook endpoint host resolution timed out",+ ));+ }+ };let mut saw_address = false;for address in resolved.by_ref() {saw_address = true;- if is_blocked_ip(address.ip()) {- return Err("webhook endpoint host resolves to a disallowed address");+ if !endpoint_scheme_policy.allows_local_addresses() && is_blocked_ip(address.ip()) {+ return Err(EndpointValidationError::InvalidConfiguration(+ "webhook endpoint host resolves to a disallowed address",+ ));}}if !saw_address {- return Err("webhook endpoint host could not be resolved");+ return Err(EndpointValidationError::ResolutionFailure(+ "webhook endpoint host could not be resolved",+ ));}Ok(url)}-fn validate_endpoint_url(value: &str) -> Result<Url, &'static str> {- let url = Url::parse(value).map_err(|_| "webhook endpoint URL is invalid")?;- if url.scheme() != "https" {- return Err("webhook endpoint URL must use HTTPS");+fn validate_endpoint_url(+ value: &str,+ endpoint_scheme_policy: WebhookEndpointSchemePolicy,+) -> Result<Url, EndpointValidationError> {+ let url = Url::parse(value).map_err(|_| {+ EndpointValidationError::InvalidConfiguration("webhook endpoint URL is invalid")+ })?;+ if !endpoint_scheme_policy.allows(url.scheme()) {+ return Err(EndpointValidationError::InvalidConfiguration(+ "webhook endpoint URL must use HTTPS",+ ));}let Some(host) = url.host_str() else {- return Err("webhook endpoint URL host is invalid");+ return Err(EndpointValidationError::InvalidConfiguration(+ "webhook endpoint URL host is invalid",+ ));};- if is_blocked_host(host) {- return Err("webhook endpoint host is not allowed");+ if !endpoint_scheme_policy.allows_local_addresses() && is_blocked_host(host) {+ return Err(EndpointValidationError::InvalidConfiguration(+ "webhook endpoint host is not allowed",+ ));}Ok(url)@@ -191,7 +258,7 @@ fn is_blocked_ip(ip: IpAddr) -> bool {}}-fn is_reserved_macro_header(name: &str) -> bool {+pub(super) fn is_reserved_macro_header(name: &str) -> bool {matches!(name.to_ascii_lowercase().as_str(),"x-macro-event" | "x-macro-event-id" | "x-macro-timestamp" | "x-macro-signature"diff --git a/rust/cloud-storage/webhook/src/outbound/http_validator_test.rs b/rust/cloud-storage/webhook/src/outbound/http_validator_test.rsindex 20bce9cb95..15f85860fe 100644--- a/rust/cloud-storage/webhook/src/outbound/http_validator_test.rs+++ b/rust/cloud-storage/webhook/src/outbound/http_validator_test.rs@@ -1,5 +1,10 @@use super::*;+#[test]+fn uses_five_second_request_timeout() {+ assert_eq!(REQUEST_TIMEOUT, Duration::from_secs(5));+}+#[test]fn builds_deterministic_validation_body() {let body = validation_body("wh_test", "evt_test").expect("body should serialize");@@ -33,28 +38,66 @@ fn signs_timestamp_and_raw_body() {#[test]fn rejects_non_https_endpoint() {- let error = validate_endpoint_url("http://example.com/webhook").expect_err("http is invalid");+ let error = validate_endpoint_url(+ "http://example.com/webhook",+ WebhookEndpointSchemePolicy::HttpsOnly,+ )+ .expect_err("http is invalid");++ assert_eq!(error.message(), "webhook endpoint URL must use HTTPS");+}- assert_eq!(error, "webhook endpoint URL must use HTTPS");+#[test]+fn allows_local_http_endpoints_when_policy_permits_them() {+ let policy = WebhookEndpointSchemePolicy::HttpAndHttps;++ for value in [+ "http://localhost/webhook",+ "http://127.0.0.1/webhook",+ "http://10.1.2.3/webhook",+ "http://[::1]/webhook",+ "http://[fe80::1]/webhook",+ ] {+ let url = validate_endpoint_url(value, policy).expect("local endpoint should be allowed");++ assert_eq!(url.scheme(), "http");+ }}#[test]fn rejects_localhost_and_private_ips() {- assert!(validate_endpoint_url("https://localhost/webhook").is_err());- assert!(validate_endpoint_url("https://127.0.0.1/webhook").is_err());- assert!(validate_endpoint_url("https://10.1.2.3/webhook").is_err());- assert!(validate_endpoint_url("https://169.254.169.254/latest/meta-data").is_err());- assert!(validate_endpoint_url("https://[::1]/webhook").is_err());- assert!(validate_endpoint_url("https://[fe80::1]/webhook").is_err());+ let policy = WebhookEndpointSchemePolicy::HttpsOnly;++ assert!(validate_endpoint_url("https://localhost/webhook", policy).is_err());+ assert!(validate_endpoint_url("https://127.0.0.1/webhook", policy).is_err());+ assert!(validate_endpoint_url("https://10.1.2.3/webhook", policy).is_err());+ assert!(validate_endpoint_url("https://169.254.169.254/latest/meta-data", policy).is_err());+ assert!(validate_endpoint_url("https://[::1]/webhook", policy).is_err());+ assert!(validate_endpoint_url("https://[fe80::1]/webhook", policy).is_err());}#[tokio::test]async fn rejects_hosts_that_resolve_to_blocked_addresses() {- let error = validate_resolved_endpoint_url("https://localhost/webhook")- .await- .expect_err("localhost is invalid");+ let error = validate_resolved_endpoint_url(+ "https://localhost/webhook",+ WebhookEndpointSchemePolicy::HttpsOnly,+ )+ .await+ .expect_err("localhost is invalid");++ assert_eq!(error.message(), "webhook endpoint host is not allowed");+}- assert_eq!(error, "webhook endpoint host is not allowed");+#[tokio::test]+async fn allows_localhost_resolution_when_policy_permits_it() {+ let url = validate_resolved_endpoint_url(+ "http://localhost/webhook",+ WebhookEndpointSchemePolicy::HttpAndHttps,+ )+ .await+ .expect("localhost should resolve");++ assert_eq!(url.host_str(), Some("localhost"));}#[test]@@ -66,7 +109,11 @@ fn blocks_ipv6_link_local_addresses() {#[test]fn accepts_public_https_endpoint() {- let url = validate_endpoint_url("https://example.com/webhook").expect("public https is valid");+ let url = validate_endpoint_url(+ "https://example.com/webhook",+ WebhookEndpointSchemePolicy::HttpsOnly,+ )+ .expect("public https is valid");assert_eq!(url.host_str(), Some("example.com"));}diff --git a/rust/cloud-storage/webhook/src/outbound/pg_delivery_repository.rs b/rust/cloud-storage/webhook/src/outbound/pg_delivery_repository.rsnew file mode 100644index 0000000000..9c20be26b6--- /dev/null+++ b/rust/cloud-storage/webhook/src/outbound/pg_delivery_repository.rs@@ -0,0 +1,655 @@+//! PostgreSQL webhook delivery repository.++#[cfg(test)]+#[path = "pg_delivery_repository_test.rs"]+mod pg_delivery_repository_test;++use crate::domain::{+ models::{+ PreparedWebhookDelivery, Webhook, WebhookDeliveryAttempt, WebhookDeliveryAttemptStatus,+ WebhookDeliveryStatus, WebhookEventQueueMessage, WebhookFilters, WebhookHeaders,+ WebhookHttpOutcomeDetails, WebhookStatus,+ },+ ports::WebhookDeliveryRepository,+};+use chrono::{DateTime, Utc};+use serde_json::Value;+use sqlx::{PgPool, Postgres, Transaction};+use std::str::FromStr;++const REDACTED_HEADER_VALUE: &str = "[REDACTED]";++/// PostgreSQL-backed implementation of [`WebhookDeliveryRepository`].+#[derive(Clone)]+pub struct PgWebhookDeliveryRepository {+ pool: PgPool,+}++impl PgWebhookDeliveryRepository {+ /// Create a webhook delivery repository backed by the supplied pool.+ pub fn new(pool: PgPool) -> Self {+ Self { pool }+ }+}++#[derive(sqlx::FromRow)]+struct PreparedDeliveryRow {+ delivery_id: String,+ delivery_status: String,+ attempt_count: i32,+ next_attempt_at: Option<DateTime<Utc>>,+ webhook_id: String,+ workspace_id: String,+ name: String,+ endpoint_url: String,+ signing_secret: String,+ headers: Value,+ webhook_status: String,+ is_valid: bool,+ created_by_user_id: String,+ created_at: DateTime<Utc>,+ updated_at: DateTime<Utc>,+ deleted_at: Option<DateTime<Utc>>,+ filters: Value,+}++#[derive(Clone, Copy)]+struct CompletionState {+ attempt_status: WebhookDeliveryAttemptStatus,+ delivery_status: WebhookDeliveryStatus,+ next_attempt_at: Option<DateTime<Utc>>,+ delivered: bool,+}++fn new_delivery_id() -> String {+ format!("whd_{}", macro_uuid::generate_uuid_v7())+}++fn new_attempt_id() -> String {+ format!("wha_{}", macro_uuid::generate_uuid_v7())+}++fn decode_error(+ column: &str,+ message: impl Into<Box<dyn std::error::Error + Send + Sync>>,+) -> sqlx::Error {+ sqlx::Error::ColumnDecode {+ index: column.to_string(),+ source: message.into(),+ }+}++fn row_to_prepared_delivery(+ row: PreparedDeliveryRow,+) -> Result<PreparedWebhookDelivery, sqlx::Error> {+ let status = WebhookDeliveryStatus::from_str(&row.delivery_status)+ .map_err(|message| decode_error("delivery_status", message))?;+ let webhook_status = WebhookStatus::from_str(&row.webhook_status)+ .map_err(|message| decode_error("webhook_status", message))?;+ let filters = serde_json::from_value::<WebhookFilters>(row.filters)+ .map_err(|error| decode_error("filters", error))?;+ let headers = serde_json::from_value::<WebhookHeaders>(row.headers)+ .map_err(|error| decode_error("headers", error))?;+ let attempt_count =+ u32::try_from(row.attempt_count).map_err(|error| decode_error("attempt_count", error))?;++ Ok(PreparedWebhookDelivery {+ delivery_id: row.delivery_id,+ webhook: Webhook {+ id: row.webhook_id,+ workspace_id: row.workspace_id,+ name: row.name,+ endpoint_url: row.endpoint_url,+ signing_secret: row.signing_secret,+ headers,+ status: webhook_status,+ is_valid: row.is_valid,+ created_by_user_id: row.created_by_user_id,+ created_at: row.created_at,+ updated_at: row.updated_at,+ deleted_at: row.deleted_at,+ filters,+ },+ status,+ attempt_count,+ next_attempt_at: row.next_attempt_at,+ })+}++fn encode_error(error: impl std::error::Error + Send + Sync + 'static) -> sqlx::Error {+ sqlx::Error::Encode(Box::new(error))+}++fn duration_milliseconds(details: &WebhookHttpOutcomeDetails) -> Result<i32, sqlx::Error> {+ i32::try_from(details.duration.as_millis()).map_err(encode_error)+}++fn response_status(details: &WebhookHttpOutcomeDetails) -> Option<i32> {+ details.response_status.map(i32::from)+}++fn response_headers(details: &WebhookHttpOutcomeDetails) -> Result<Option<Value>, sqlx::Error> {+ details+ .response_headers_redacted+ .as_ref()+ .map(serde_json::to_value)+ .transpose()+ .map_err(encode_error)+}++async fn interrupt_current_attempt(+ transaction: &mut Transaction<'_, Postgres>,+ delivery_id: &str,+ webhook_id: &str,+ attempt_count: i32,+) -> Result<(), sqlx::Error> {+ if attempt_count == 0 {+ return Ok(());+ }++ let interrupted_status = WebhookDeliveryAttemptStatus::Interrupted.as_str();+ let in_progress_status = WebhookDeliveryAttemptStatus::InProgress.as_str();+ let interrupted = sqlx::query!(+ r#"+ UPDATE webhook_delivery_attempt+ SET+ status = $3,+ completed_at = now()+ WHERE webhook_delivery_id = $1+ AND attempt_number = $2+ AND status = $4+ RETURNING completed_at AS "completed_at!"+ "#,+ delivery_id,+ attempt_count,+ interrupted_status,+ in_progress_status+ )+ .fetch_optional(&mut **transaction)+ .await?;++ if let Some(interrupted) = interrupted {+ sqlx::query!(+ r#"+ UPDATE webhook+ SET+ last_failure_at = GREATEST(COALESCE(last_failure_at, $2), $2),+ updated_at = GREATEST(updated_at, $2)+ WHERE id = $1+ "#,+ webhook_id,+ interrupted.completed_at+ )+ .execute(&mut **transaction)+ .await?;+ }++ Ok(())+}++async fn complete_attempt(+ pool: &PgPool,+ attempt: &WebhookDeliveryAttempt,+ details: &WebhookHttpOutcomeDetails,+ completion: CompletionState,+) -> Result<(), sqlx::Error> {+ let attempt_number = i32::try_from(attempt.attempt_number).map_err(encode_error)?;+ let duration_ms = duration_milliseconds(details)?;+ let response_status = response_status(details);+ let response_headers_redacted = response_headers(details)?;+ let mut transaction = pool.begin().await?;++ let delivery = sqlx::query!(+ r#"+ SELECT webhook_id, status, attempt_count+ FROM webhook_delivery+ WHERE id = $1+ FOR UPDATE+ "#,+ attempt.delivery_id+ )+ .fetch_optional(&mut *transaction)+ .await?+ .ok_or(sqlx::Error::RowNotFound)?;++ if delivery.status != WebhookDeliveryStatus::InProgress.as_str()+ || delivery.attempt_count != attempt_number+ {+ return Err(sqlx::Error::RowNotFound);+ }++ let attempt_status = completion.attempt_status.as_str();+ let in_progress_attempt_status = WebhookDeliveryAttemptStatus::InProgress.as_str();+ let completed = sqlx::query!(+ r#"+ UPDATE webhook_delivery_attempt+ SET+ status = $4,+ completed_at = now(),+ duration_ms = $5,+ response_status = $6,+ response_headers_redacted = $7,+ response_body_preview = $8,+ error_kind = $9,+ error_message = $10+ WHERE id = $1+ AND webhook_delivery_id = $2+ AND attempt_number = $3+ AND status = $11+ RETURNING completed_at AS "completed_at!"+ "#,+ attempt.attempt_id,+ attempt.delivery_id,+ attempt_number,+ attempt_status,+ duration_ms,+ response_status,+ response_headers_redacted,+ details.response_body_preview,+ details.error_kind,+ details.error_message,+ in_progress_attempt_status+ )+ .fetch_optional(&mut *transaction)+ .await?+ .ok_or(sqlx::Error::RowNotFound)?;++ let delivery_status = completion.delivery_status.as_str();+ let delivered_at = completion.delivered.then_some(completed.completed_at);+ let updated = sqlx::query!(+ r#"+ UPDATE webhook_delivery+ SET+ status = $2,+ next_attempt_at = $3,+ delivered_at = $4,+ updated_at = $5+ WHERE id = $1+ AND status = $6+ AND attempt_count = $7+ "#,+ attempt.delivery_id,+ delivery_status,+ completion.next_attempt_at,+ delivered_at,+ completed.completed_at,+ WebhookDeliveryStatus::InProgress.as_str(),+ attempt_number+ )+ .execute(&mut *transaction)+ .await?;++ if updated.rows_affected() != 1 {+ return Err(sqlx::Error::RowNotFound);+ }++ if completion.delivered {+ sqlx::query!(+ r#"+ UPDATE webhook+ SET+ last_success_at = GREATEST(COALESCE(last_success_at, $2), $2),+ updated_at = GREATEST(updated_at, $2)+ WHERE id = $1+ "#,+ delivery.webhook_id,+ completed.completed_at+ )+ .execute(&mut *transaction)+ .await?;+ } else {+ sqlx::query!(+ r#"+ UPDATE webhook+ SET+ last_failure_at = GREATEST(COALESCE(last_failure_at, $2), $2),+ updated_at = GREATEST(updated_at, $2)+ WHERE id = $1+ "#,+ delivery.webhook_id,+ completed.completed_at+ )+ .execute(&mut *transaction)+ .await?;+ }++ transaction.commit().await+}++impl WebhookDeliveryRepository for PgWebhookDeliveryRepository {+ type Err = sqlx::Error;++ #[tracing::instrument(skip(self, message), fields(webhook_id = %message.webhook_id, event_id = %message.event.event_id), err)]+ async fn prepare_delivery(+ &self,+ message: &WebhookEventQueueMessage,+ ) -> Result<Option<PreparedWebhookDelivery>, Self::Err> {+ let mut transaction = self.pool.begin().await?;+ let delivery_id = new_delivery_id();+ let delivery_status = WebhookDeliveryStatus::Queued.as_str();+ let event_schema_version = i32::from(message.event.schema_version);++ sqlx::query!(+ r#"+ INSERT INTO webhook_delivery (+ id,+ webhook_id,+ event_id,+ event,+ event_schema_version,+ event_occurred_at,+ event_entity_type,+ event_entity_id,+ event_ordering_key,+ status,+ request_url,+ request_headers_redacted,+ request_body+ )+ SELECT+ $1,+ w.id,+ $3,+ $4,+ $5,+ $6,+ $7,+ $8,+ $9,+ $10,+ w.endpoint_url,+ COALESCE(+ (+ SELECT jsonb_object_agg(header_name, $11::TEXT)+ FROM jsonb_object_keys(w.headers) AS header_name+ ),+ '{}'::JSONB+ ),+ $12+ FROM webhook w+ WHERE w.id = $2+ ON CONFLICT (webhook_id, event_id) DO NOTHING+ "#,+ delivery_id,+ message.webhook_id,+ message.event.event_id,+ message.event.event_name,+ event_schema_version,+ message.event.occurred_at,+ message.event.entity_type,+ message.event.entity_id,+ message.event.ordering_key,+ delivery_status,+ REDACTED_HEADER_VALUE,+ message.event.broker_envelope+ )+ .execute(&mut *transaction)+ .await?;++ let row = sqlx::query_as!(+ PreparedDeliveryRow,+ r#"+ SELECT+ d.id AS delivery_id,+ d.status AS delivery_status,+ d.attempt_count,+ d.next_attempt_at,+ w.id AS webhook_id,+ w.workspace_id,+ w.name,+ w.endpoint_url,+ w.signing_secret,+ w.headers,+ w.status AS webhook_status,+ w.is_valid,+ w.created_by_user_id,+ w.created_at,+ w.updated_at,+ w.deleted_at,+ w.filters+ FROM webhook_delivery d+ JOIN webhook w ON w.id = d.webhook_id+ WHERE d.webhook_id = $1+ AND d.event_id = $2+ "#,+ message.webhook_id,+ message.event.event_id+ )+ .fetch_optional(&mut *transaction)+ .await?;++ transaction.commit().await?;+ row.map(row_to_prepared_delivery).transpose()+ }++ #[tracing::instrument(skip(self), err)]+ async fn cancel_delivery(&self, delivery_id: &str) -> Result<(), Self::Err> {+ let mut transaction = self.pool.begin().await?;+ let delivery = sqlx::query!(+ r#"+ SELECT webhook_id, status, attempt_count+ FROM webhook_delivery+ WHERE id = $1+ FOR UPDATE+ "#,+ delivery_id+ )+ .fetch_optional(&mut *transaction)+ .await?;++ let Some(delivery) = delivery else {+ transaction.commit().await?;+ return Ok(());+ };++ let status = WebhookDeliveryStatus::from_str(&delivery.status)+ .map_err(|message| decode_error("status", message))?;+ if status.is_terminal() {+ transaction.commit().await?;+ return Ok(());+ }++ if status == WebhookDeliveryStatus::InProgress {+ interrupt_current_attempt(+ &mut transaction,+ delivery_id,+ &delivery.webhook_id,+ delivery.attempt_count,+ )+ .await?;+ }++ sqlx::query!(+ r#"+ UPDATE webhook_delivery+ SET+ status = $2,+ next_attempt_at = NULL,+ updated_at = now()+ WHERE id = $1+ "#,+ delivery_id,+ WebhookDeliveryStatus::Canceled.as_str()+ )+ .execute(&mut *transaction)+ .await?;++ transaction.commit().await+ }++ #[tracing::instrument(skip(self), err)]+ async fn begin_attempt(+ &self,+ delivery_id: &str,+ ) -> Result<Option<WebhookDeliveryAttempt>, Self::Err> {+ let mut transaction = self.pool.begin().await?;+ let delivery = sqlx::query!(+ r#"+ SELECT+ webhook_id,+ status,+ attempt_count,+ COALESCE(next_attempt_at <= now(), true) AS "is_due!"+ FROM webhook_delivery+ WHERE id = $1+ FOR UPDATE+ "#,+ delivery_id+ )+ .fetch_optional(&mut *transaction)+ .await?;++ let Some(delivery) = delivery else {+ transaction.commit().await?;+ return Ok(None);+ };++ let status = WebhookDeliveryStatus::from_str(&delivery.status)+ .map_err(|message| decode_error("status", message))?;+ if status.is_terminal() || !delivery.is_due {+ transaction.commit().await?;+ return Ok(None);+ }++ if status == WebhookDeliveryStatus::InProgress {+ interrupt_current_attempt(+ &mut transaction,+ delivery_id,+ &delivery.webhook_id,+ delivery.attempt_count,+ )+ .await?;+ }++ let started = sqlx::query!(+ r#"+ UPDATE webhook_delivery+ SET+ status = $2,+ attempt_count = attempt_count + 1,+ first_attempt_at = COALESCE(first_attempt_at, now()),+ last_attempt_at = now(),+ next_attempt_at = NULL,+ updated_at = now()+ WHERE id = $1+ RETURNING+ attempt_count,+ last_attempt_at AS "started_at!"+ "#,+ delivery_id,+ WebhookDeliveryStatus::InProgress.as_str()+ )+ .fetch_one(&mut *transaction)+ .await?;++ let attempt_id = new_attempt_id();+ sqlx::query!(+ r#"+ INSERT INTO webhook_delivery_attempt (+ id,+ webhook_delivery_id,+ attempt_number,+ status,+ started_at+ )+ VALUES ($1, $2, $3, $4, $5)+ "#,+ attempt_id,+ delivery_id,+ started.attempt_count,+ WebhookDeliveryAttemptStatus::InProgress.as_str(),+ started.started_at+ )+ .execute(&mut *transaction)+ .await?;++ transaction.commit().await?;+ Ok(Some(WebhookDeliveryAttempt {+ attempt_id,+ delivery_id: delivery_id.to_string(),+ attempt_number: u32::try_from(started.attempt_count)+ .map_err(|error| decode_error("attempt_count", error))?,+ }))+ }++ #[tracing::instrument(skip(self, details), err)]+ async fn record_success(+ &self,+ attempt: &WebhookDeliveryAttempt,+ details: &WebhookHttpOutcomeDetails,+ ) -> Result<(), Self::Err> {+ complete_attempt(+ &self.pool,+ attempt,+ details,+ CompletionState {+ attempt_status: WebhookDeliveryAttemptStatus::Succeeded,+ delivery_status: WebhookDeliveryStatus::Delivered,+ next_attempt_at: None,+ delivered: true,+ },+ )+ .await+ }++ #[tracing::instrument(skip(self, details), err)]+ async fn record_retryable_failure(+ &self,+ attempt: &WebhookDeliveryAttempt,+ details: &WebhookHttpOutcomeDetails,+ next_attempt_at: DateTime<Utc>,+ ) -> Result<(), Self::Err> {+ complete_attempt(+ &self.pool,+ attempt,+ details,+ CompletionState {+ attempt_status: WebhookDeliveryAttemptStatus::RetryableFailure,+ delivery_status: WebhookDeliveryStatus::RetryScheduled,+ next_attempt_at: Some(next_attempt_at),+ delivered: false,+ },+ )+ .await+ }++ #[tracing::instrument(skip(self, details), err)]+ async fn record_permanent_failure(+ &self,+ attempt: &WebhookDeliveryAttempt,+ details: &WebhookHttpOutcomeDetails,+ ) -> Result<(), Self::Err> {+ complete_attempt(+ &self.pool,+ attempt,+ details,+ CompletionState {+ attempt_status: WebhookDeliveryAttemptStatus::PermanentFailure,+ delivery_status: WebhookDeliveryStatus::PermanentlyFailed,+ next_attempt_at: None,+ delivered: false,+ },+ )+ .await+ }++ #[tracing::instrument(skip(self, details), err)]+ async fn record_exhaustion(+ &self,+ attempt: &WebhookDeliveryAttempt,+ details: &WebhookHttpOutcomeDetails,+ ) -> Result<(), Self::Err> {+ complete_attempt(+ &self.pool,+ attempt,+ details,+ CompletionState {+ attempt_status: WebhookDeliveryAttemptStatus::Exhausted,+ delivery_status: WebhookDeliveryStatus::Exhausted,+ next_attempt_at: None,+ delivered: false,+ },+ )+ .await+ }+}diff --git a/rust/cloud-storage/webhook/src/outbound/pg_delivery_repository_test.rs b/rust/cloud-storage/webhook/src/outbound/pg_delivery_repository_test.rsnew file mode 100644index 0000000000..f929b62add--- /dev/null+++ b/rust/cloud-storage/webhook/src/outbound/pg_delivery_repository_test.rs@@ -0,0 +1,616 @@+use super::*;+use crate::domain::{+ models::{+ NormalizedWebhookEvent, WebhookDeliveryAttemptStatus, WebhookEventQueueMessage,+ WebhookHeaders,+ },+ ports::WebhookDeliveryRepository,+};+use macro_db_migrator::MACRO_DB_MIGRATIONS;+use serde_json::json;+use sqlx::PgPool;+use std::{collections::BTreeMap, time::Duration};++const WEBHOOK_ID: &str = "wh_delivery_test";+const USER_ID: &str = "macro|[email protected]";+const ENDPOINT_URL: &str = "https://example.com/webhooks/original";+const SIGNING_SECRET: &str = "secret-never-snapshotted";++async fn insert_webhook(pool: &PgPool, deleted: bool) -> anyhow::Result<()> {+ let deleted_at = deleted.then(Utc::now);+ sqlx::query!(+ r#"+ INSERT INTO webhook (+ id,+ workspace_id,+ name,+ endpoint_url,+ signing_secret,+ headers,+ filters,+ status,+ is_valid,+ created_by_user_id,+ deleted_at+ )+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, true, $9, $10)+ "#,+ WEBHOOK_ID,+ USER_ID,+ "Delivery test",+ ENDPOINT_URL,+ SIGNING_SECRET,+ json!({+ "Authorization": "Bearer private",+ "X-Custom": "also private",+ }),+ json!([{"events": ["document.created"]}]),+ WebhookStatus::Active.as_str(),+ USER_ID,+ deleted_at+ )+ .execute(pool)+ .await?;+ Ok(())+}++fn message(event_id: &str, body_marker: &str) -> WebhookEventQueueMessage {+ WebhookEventQueueMessage::new(+ WEBHOOK_ID.to_string(),+ NormalizedWebhookEvent {+ event_id: event_id.to_string(),+ schema_version: 1,+ event_name: "document.created".to_string(),+ entity_type: "document".to_string(),+ entity_id: "doc_123".to_string(),+ ordering_key: "doc_123".to_string(),+ occurred_at: Utc::now(),+ broker_envelope: json!({+ "event_id": event_id,+ "marker": body_marker,+ "metadata": {"private": true},+ }),+ },+ )+}++fn outcome_details() -> WebhookHttpOutcomeDetails {+ WebhookHttpOutcomeDetails {+ duration: Duration::from_millis(125),+ response_status: Some(503),+ response_headers_redacted: Some(BTreeMap::from([+ ("Content-Type".to_string(), "[REDACTED]".to_string()),+ ("X-Request-Id".to_string(), "[REDACTED]".to_string()),+ ])),+ response_body_preview: Some("temporarily unavailable".to_string()),+ error_kind: Some("http_status".to_string()),+ error_message: Some("webhook returned HTTP 503".to_string()),+ }+}++async fn prepare(+ repository: &PgWebhookDeliveryRepository,+ event_id: &str,+) -> anyhow::Result<PreparedWebhookDelivery> {+ repository+ .prepare_delivery(&message(event_id, "original"))+ .await?+ .ok_or_else(|| anyhow::anyhow!("webhook should exist"))+}++async fn begin(+ repository: &PgWebhookDeliveryRepository,+ delivery_id: &str,+) -> anyhow::Result<WebhookDeliveryAttempt> {+ repository+ .begin_attempt(delivery_id)+ .await?+ .ok_or_else(|| anyhow::anyhow!("delivery should be ready"))+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn prepare_is_idempotent_and_snapshots_redacted_request_data(+ pool: PgPool,+) -> anyhow::Result<()> {+ insert_webhook(&pool, false).await?;+ let repository = PgWebhookDeliveryRepository::new(pool.clone());+ let original_message = message("evt_idempotent", "original");++ let first = repository+ .prepare_delivery(&original_message)+ .await?+ .expect("prepared delivery");+ assert!(first.delivery_id.starts_with("whd_"));+ assert_eq!(first.status, WebhookDeliveryStatus::Queued);+ assert_eq!(first.attempt_count, 0);+ assert_eq!(first.webhook.signing_secret, SIGNING_SECRET);++ sqlx::query!(+ r#"+ UPDATE webhook+ SET+ endpoint_url = $2,+ headers = $3,+ updated_at = now()+ WHERE id = $1+ "#,+ WEBHOOK_ID,+ "https://example.com/webhooks/current",+ json!({"X-Current": "new private value"})+ )+ .execute(&pool)+ .await?;++ let current = repository+ .prepare_delivery(&message("evt_idempotent", "replacement"))+ .await?+ .expect("existing delivery");+ assert_eq!(current.delivery_id, first.delivery_id);+ assert_eq!(+ current.webhook.endpoint_url,+ "https://example.com/webhooks/current"+ );+ assert_eq!(+ current.webhook.headers,+ WebhookHeaders::from([("X-Current".to_string(), "new private value".to_string())])+ );++ let stored = sqlx::query!(+ r#"+ SELECT+ request_url,+ request_headers_redacted,+ request_body,+ event,+ event_schema_version,+ event_entity_type,+ event_entity_id,+ event_ordering_key+ FROM webhook_delivery+ WHERE webhook_id = $1 AND event_id = $2+ "#,+ WEBHOOK_ID,+ "evt_idempotent"+ )+ .fetch_one(&pool)+ .await?;+ let delivery_count = sqlx::query_scalar!(+ "SELECT COUNT(*) FROM webhook_delivery WHERE webhook_id = $1 AND event_id = $2",+ WEBHOOK_ID,+ "evt_idempotent"+ )+ .fetch_one(&pool)+ .await?;++ assert_eq!(delivery_count, Some(1));+ assert_eq!(stored.request_url, ENDPOINT_URL);+ assert_eq!(+ stored.request_headers_redacted,+ json!({+ "Authorization": REDACTED_HEADER_VALUE,+ "X-Custom": REDACTED_HEADER_VALUE,+ })+ );+ assert_eq!(stored.request_body, original_message.event.broker_envelope);+ assert_eq!(stored.event, "document.created");+ assert_eq!(stored.event_schema_version, 1);+ assert_eq!(stored.event_entity_type, "document");+ assert_eq!(stored.event_entity_id, "doc_123");+ assert_eq!(stored.event_ordering_key, "doc_123");+ Ok(())+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn prepare_returns_none_for_a_missing_webhook(pool: PgPool) -> anyhow::Result<()> {+ let repository = PgWebhookDeliveryRepository::new(pool.clone());++ assert!(+ repository+ .prepare_delivery(&message("evt_missing", "original"))+ .await?+ .is_none()+ );+ assert_eq!(+ sqlx::query_scalar!("SELECT COUNT(*) FROM webhook_delivery")+ .fetch_one(&pool)+ .await?,+ Some(0)+ );+ Ok(())+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn retry_scheduling_blocks_early_attempts_and_attempt_numbers_increase(+ pool: PgPool,+) -> anyhow::Result<()> {+ insert_webhook(&pool, false).await?;+ let repository = PgWebhookDeliveryRepository::new(pool.clone());+ let prepared = prepare(&repository, "evt_retry").await?;+ let first_attempt = begin(&repository, &prepared.delivery_id).await?;+ assert!(first_attempt.attempt_id.starts_with("wha_"));+ assert_eq!(first_attempt.attempt_number, 1);++ let next_attempt_at = Utc::now() + chrono::Duration::hours(1);+ repository+ .record_retryable_failure(&first_attempt, &outcome_details(), next_attempt_at)+ .await?;++ let scheduled = repository+ .prepare_delivery(&message("evt_retry", "duplicate"))+ .await?+ .expect("scheduled delivery");+ assert_eq!(scheduled.status, WebhookDeliveryStatus::RetryScheduled);+ assert_eq!(scheduled.attempt_count, 1);+ assert!(+ scheduled+ .next_attempt_at+ .is_some_and(|due| due > Utc::now())+ );+ assert!(+ repository+ .begin_attempt(&prepared.delivery_id)+ .await?+ .is_none()+ );++ sqlx::query!(+ r#"+ UPDATE webhook_delivery+ SET next_attempt_at = now() - INTERVAL '1 second'+ WHERE id = $1+ "#,+ prepared.delivery_id+ )+ .execute(&pool)+ .await?;++ let second_attempt = begin(&repository, &prepared.delivery_id).await?;+ assert_eq!(second_attempt.attempt_number, 2);+ assert_ne!(second_attempt.attempt_id, first_attempt.attempt_id);++ let state = sqlx::query!(+ r#"+ SELECT attempt_count, first_attempt_at, last_attempt_at, next_attempt_at+ FROM webhook_delivery+ WHERE id = $1+ "#,+ prepared.delivery_id+ )+ .fetch_one(&pool)+ .await?;+ let attempts = sqlx::query!(+ r#"+ SELECT attempt_number, status+ FROM webhook_delivery_attempt+ WHERE webhook_delivery_id = $1+ ORDER BY attempt_number+ "#,+ prepared.delivery_id+ )+ .fetch_all(&pool)+ .await?;+ let aggregate = sqlx::query!(+ "SELECT last_failure_at FROM webhook WHERE id = $1",+ WEBHOOK_ID+ )+ .fetch_one(&pool)+ .await?;++ assert_eq!(state.attempt_count, 2);+ assert!(state.first_attempt_at.is_some());+ assert!(state.last_attempt_at >= state.first_attempt_at);+ assert!(state.next_attempt_at.is_none());+ assert_eq!(attempts.len(), 2);+ assert_eq!(attempts[0].attempt_number, 1);+ assert_eq!(+ attempts[0].status,+ WebhookDeliveryAttemptStatus::RetryableFailure.as_str()+ );+ assert_eq!(attempts[1].attempt_number, 2);+ assert_eq!(+ attempts[1].status,+ WebhookDeliveryAttemptStatus::InProgress.as_str()+ );+ assert!(aggregate.last_failure_at.is_some());+ Ok(())+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn soft_deleted_webhook_delivery_can_be_canceled_without_an_attempt(+ pool: PgPool,+) -> anyhow::Result<()> {+ insert_webhook(&pool, true).await?;+ let repository = PgWebhookDeliveryRepository::new(pool.clone());+ let prepared = prepare(&repository, "evt_canceled").await?;+ assert!(prepared.webhook.deleted_at.is_some());++ repository.cancel_delivery(&prepared.delivery_id).await?;+ repository.cancel_delivery(&prepared.delivery_id).await?;++ let delivery = sqlx::query!(+ "SELECT status, attempt_count FROM webhook_delivery WHERE id = $1",+ prepared.delivery_id+ )+ .fetch_one(&pool)+ .await?;+ let attempt_count = sqlx::query_scalar!(+ "SELECT COUNT(*) FROM webhook_delivery_attempt WHERE webhook_delivery_id = $1",+ prepared.delivery_id+ )+ .fetch_one(&pool)+ .await?;+ let aggregate = sqlx::query!(+ "SELECT last_success_at, last_failure_at FROM webhook WHERE id = $1",+ WEBHOOK_ID+ )+ .fetch_one(&pool)+ .await?;++ assert_eq!(delivery.status, WebhookDeliveryStatus::Canceled.as_str());+ assert_eq!(delivery.attempt_count, 0);+ assert_eq!(attempt_count, Some(0));+ assert!(aggregate.last_success_at.is_none());+ assert!(aggregate.last_failure_at.is_none());+ assert!(+ repository+ .begin_attempt(&prepared.delivery_id)+ .await?+ .is_none()+ );+ Ok(())+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn success_is_terminal_and_updates_success_aggregate_atomically(+ pool: PgPool,+) -> anyhow::Result<()> {+ insert_webhook(&pool, false).await?;+ let repository = PgWebhookDeliveryRepository::new(pool.clone());+ let prepared = prepare(&repository, "evt_success").await?;+ let attempt = begin(&repository, &prepared.delivery_id).await?;+ let mut details = outcome_details();+ details.response_status = Some(204);+ details.error_kind = None;+ details.error_message = None;++ repository.record_success(&attempt, &details).await?;++ let duplicate = repository+ .prepare_delivery(&message("evt_success", "redelivery"))+ .await?+ .expect("terminal delivery");+ assert_eq!(duplicate.delivery_id, prepared.delivery_id);+ assert_eq!(duplicate.status, WebhookDeliveryStatus::Delivered);+ assert!(+ repository+ .begin_attempt(&prepared.delivery_id)+ .await?+ .is_none()+ );++ let state = sqlx::query!(+ r#"+ SELECT+ d.status,+ d.delivered_at,+ d.attempt_count,+ a.status AS attempt_status,+ a.completed_at,+ w.last_success_at,+ w.last_failure_at+ FROM webhook_delivery d+ JOIN webhook_delivery_attempt a ON a.webhook_delivery_id = d.id+ JOIN webhook w ON w.id = d.webhook_id+ WHERE d.id = $1+ "#,+ prepared.delivery_id+ )+ .fetch_one(&pool)+ .await?;++ assert_eq!(state.status, WebhookDeliveryStatus::Delivered.as_str());+ assert_eq!(state.attempt_count, 1);+ assert_eq!(+ state.attempt_status,+ WebhookDeliveryAttemptStatus::Succeeded.as_str()+ );+ assert!(state.delivered_at.is_some());+ assert_eq!(state.delivered_at, state.completed_at);+ assert_eq!(state.delivered_at, state.last_success_at);+ assert!(state.last_failure_at.is_none());+ Ok(())+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn permanent_failure_is_terminal_and_updates_failure_aggregate(+ pool: PgPool,+) -> anyhow::Result<()> {+ insert_webhook(&pool, false).await?;+ let repository = PgWebhookDeliveryRepository::new(pool.clone());+ let prepared = prepare(&repository, "evt_permanent").await?;+ let attempt = begin(&repository, &prepared.delivery_id).await?;++ repository+ .record_permanent_failure(&attempt, &outcome_details())+ .await?;++ let state = sqlx::query!(+ r#"+ SELECT+ d.status,+ d.delivered_at,+ a.status AS attempt_status,+ a.completed_at,+ a.duration_ms,+ a.response_status,+ a.response_headers_redacted,+ a.response_body_preview,+ a.error_kind,+ a.error_message,+ w.last_success_at,+ w.last_failure_at+ FROM webhook_delivery d+ JOIN webhook_delivery_attempt a ON a.webhook_delivery_id = d.id+ JOIN webhook w ON w.id = d.webhook_id+ WHERE d.id = $1+ "#,+ prepared.delivery_id+ )+ .fetch_one(&pool)+ .await?;++ assert_eq!(+ state.status,+ WebhookDeliveryStatus::PermanentlyFailed.as_str()+ );+ assert_eq!(+ state.attempt_status,+ WebhookDeliveryAttemptStatus::PermanentFailure.as_str()+ );+ assert!(state.delivered_at.is_none());+ assert_eq!(state.duration_ms, Some(125));+ assert_eq!(state.response_status, Some(503));+ assert_eq!(+ state.response_headers_redacted,+ Some(json!({+ "Content-Type": "[REDACTED]",+ "X-Request-Id": "[REDACTED]",+ }))+ );+ assert_eq!(+ state.response_body_preview.as_deref(),+ Some("temporarily unavailable")+ );+ assert_eq!(state.error_kind.as_deref(), Some("http_status"));+ assert_eq!(+ state.error_message.as_deref(),+ Some("webhook returned HTTP 503")+ );+ assert_eq!(state.completed_at, state.last_failure_at);+ assert!(state.last_success_at.is_none());+ assert!(+ repository+ .begin_attempt(&prepared.delivery_id)+ .await?+ .is_none()+ );+ Ok(())+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn exhaustion_is_terminal_and_updates_failure_aggregate(pool: PgPool) -> anyhow::Result<()> {+ insert_webhook(&pool, false).await?;+ let repository = PgWebhookDeliveryRepository::new(pool.clone());+ let prepared = prepare(&repository, "evt_exhausted").await?;+ let attempt = begin(&repository, &prepared.delivery_id).await?;++ repository+ .record_exhaustion(&attempt, &outcome_details())+ .await?;++ let state = sqlx::query!(+ r#"+ SELECT+ d.status,+ d.next_attempt_at,+ a.status AS attempt_status,+ a.completed_at,+ w.last_failure_at+ FROM webhook_delivery d+ JOIN webhook_delivery_attempt a ON a.webhook_delivery_id = d.id+ JOIN webhook w ON w.id = d.webhook_id+ WHERE d.id = $1+ "#,+ prepared.delivery_id+ )+ .fetch_one(&pool)+ .await?;++ assert_eq!(state.status, WebhookDeliveryStatus::Exhausted.as_str());+ assert_eq!(+ state.attempt_status,+ WebhookDeliveryAttemptStatus::Exhausted.as_str()+ );+ assert!(state.next_attempt_at.is_none());+ assert_eq!(state.completed_at, state.last_failure_at);+ assert!(+ repository+ .begin_attempt(&prepared.delivery_id)+ .await?+ .is_none()+ );+ Ok(())+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn interrupted_attempt_is_recorded_and_recovered_monotonically(+ pool: PgPool,+) -> anyhow::Result<()> {+ insert_webhook(&pool, false).await?;+ let repository = PgWebhookDeliveryRepository::new(pool.clone());+ let prepared = prepare(&repository, "evt_interrupted").await?;+ let interrupted_attempt = begin(&repository, &prepared.delivery_id).await?;++ let recovered_attempt = begin(&repository, &prepared.delivery_id).await?;+ assert_eq!(interrupted_attempt.attempt_number, 1);+ assert_eq!(recovered_attempt.attempt_number, 2);++ let attempts = sqlx::query!(+ r#"+ SELECT attempt_number, status, completed_at+ FROM webhook_delivery_attempt+ WHERE webhook_delivery_id = $1+ ORDER BY attempt_number+ "#,+ prepared.delivery_id+ )+ .fetch_all(&pool)+ .await?;+ let aggregate = sqlx::query!(+ "SELECT last_failure_at FROM webhook WHERE id = $1",+ WEBHOOK_ID+ )+ .fetch_one(&pool)+ .await?;++ assert_eq!(attempts.len(), 2);+ assert_eq!(+ attempts[0].status,+ WebhookDeliveryAttemptStatus::Interrupted.as_str()+ );+ assert!(attempts[0].completed_at.is_some());+ assert_eq!(attempts[0].completed_at, aggregate.last_failure_at);+ assert_eq!(+ attempts[1].status,+ WebhookDeliveryAttemptStatus::InProgress.as_str()+ );+ assert!(attempts[1].completed_at.is_none());++ let stale_result = repository+ .record_success(&interrupted_attempt, &outcome_details())+ .await;+ assert!(matches!(stale_result, Err(sqlx::Error::RowNotFound)));++ let before_completion = sqlx::query!(+ "SELECT status, attempt_count FROM webhook_delivery WHERE id = $1",+ prepared.delivery_id+ )+ .fetch_one(&pool)+ .await?;+ assert_eq!(+ before_completion.status,+ WebhookDeliveryStatus::InProgress.as_str()+ );+ assert_eq!(before_completion.attempt_count, 2);++ repository+ .record_success(&recovered_attempt, &outcome_details())+ .await?;+ let recovered = repository+ .prepare_delivery(&message("evt_interrupted", "redelivery"))+ .await?+ .expect("recovered delivery");+ assert_eq!(recovered.status, WebhookDeliveryStatus::Delivered);+ assert_eq!(recovered.attempt_count, 2);+ Ok(())+}diff --git a/rust/cloud-storage/webhook/src/outbound/pg_repository.rs b/rust/cloud-storage/webhook/src/outbound/pg_repository.rsindex 9e8e333504..0a8de938ae 100644--- a/rust/cloud-storage/webhook/src/outbound/pg_repository.rs+++ b/rust/cloud-storage/webhook/src/outbound/pg_repository.rs@@ -8,7 +8,7 @@ use crate::domain::{CreateWebhookRequest, PatchWebhookRequest, Webhook, WebhookFilters, WebhookHeaders,WebhookId, WebhookStatus,},- ports::WebhookRepo,+ ports::{WebhookRepo, WebhookWorkspaceResolver},};use macro_user_id::user_id::MacroUserIdStr;use serde_json::Value;@@ -115,6 +115,44 @@ async fn fetch_webhook(pool: &PgPool, webhook_id: &str) -> Result<Option<Webhookrow.map(row_to_webhook).transpose()}+impl WebhookWorkspaceResolver for PgRepository {+ type Err = sqlx::Error;++ #[tracing::instrument(skip(self, people), err)]+ async fn resolve_workspace_ids(+ &self,+ people: Vec<MacroUserIdStr<'static>>,+ ) -> Result<Vec<String>, Self::Err> {+ let mut workspace_ids = people+ .iter()+ .map(|person| person.as_ref().to_string())+ .collect::<Vec<_>>();+ workspace_ids.sort_unstable();+ workspace_ids.dedup();++ if workspace_ids.is_empty() {+ return Ok(workspace_ids);+ }++ let team_workspace_ids = sqlx::query_scalar!(+ r#"+ SELECT DISTINCT team_id::text AS "team_id!"+ FROM team_user+ WHERE user_id = ANY($1)+ ORDER BY team_id::text+ "#,+ &workspace_ids+ )+ .fetch_all(&self.pool)+ .await?;++ workspace_ids.extend(team_workspace_ids);+ workspace_ids.sort_unstable();+ workspace_ids.dedup();+ Ok(workspace_ids)+ }+}+impl WebhookRepo for PgRepository {type Err = sqlx::Error;diff --git a/rust/cloud-storage/webhook/src/outbound/pg_repository_test.rs b/rust/cloud-storage/webhook/src/outbound/pg_repository_test.rsindex d9bba1d9b6..295450216a 100644--- a/rust/cloud-storage/webhook/src/outbound/pg_repository_test.rs+++ b/rust/cloud-storage/webhook/src/outbound/pg_repository_test.rs@@ -1,7 +1,7 @@use super::*;use crate::domain::{models::{WebhookFilter, WebhookStatus},- ports::WebhookRepo,+ ports::{WebhookRepo, WebhookWorkspaceResolver},};use macro_db_migrator::MACRO_DB_MIGRATIONS;use macro_user_id::user_id::MacroUserIdStr;@@ -9,11 +9,21 @@ use serde_json::json;use sqlx::{PgPool, types::Uuid};const USER_ID: &str = "macro|[email protected]";+const SECOND_USER_ID: &str = "macro|[email protected]";const TEAM_ID: &str = "11111111-1111-1111-1111-111111111111";+const UNRELATED_TEAM_ID: &str = "22222222-2222-2222-2222-222222222222";const OTHER_WORKSPACE_ID: &str = "workspace_other";+fn macro_user_id(user_id: &str) -> MacroUserIdStr<'static> {+ MacroUserIdStr::try_from(user_id.to_string()).expect("valid macro user id")+}+fn user_id() -> MacroUserIdStr<'static> {- MacroUserIdStr::try_from(USER_ID.to_string()).expect("valid macro user id")+ macro_user_id(USER_ID)+}++fn second_user_id() -> MacroUserIdStr<'static> {+ macro_user_id(SECOND_USER_ID)}fn create_request() -> CreateWebhookRequest {@@ -29,7 +39,7 @@ fn create_request() -> CreateWebhookRequest {}}-async fn insert_user(pool: &PgPool) -> anyhow::Result<()> {+async fn insert_user_with_id(pool: &PgPool, user_id: &str, email: &str) -> anyhow::Result<()> {let macro_user_id = macro_uuid::generate_uuid_v7();let stripe_customer_id = format!("stripe_{macro_user_id}");sqlx::query!(@@ -39,8 +49,8 @@ async fn insert_user(pool: &PgPool) -> anyhow::Result<()> {ON CONFLICT (id) DO NOTHING"#,macro_user_id,- "[email protected]",- "[email protected]",+ email,+ email,stripe_customer_id).execute(pool)@@ -52,8 +62,8 @@ async fn insert_user(pool: &PgPool) -> anyhow::Result<()> {VALUES ($1, $2, $3)ON CONFLICT (id) DO NOTHING"#,- USER_ID,- "[email protected]",+ user_id,+ email,macro_user_id).execute(pool)@@ -62,6 +72,33 @@ async fn insert_user(pool: &PgPool) -> anyhow::Result<()> {Ok(())}+async fn insert_user(pool: &PgPool) -> anyhow::Result<()> {+ insert_user_with_id(pool, USER_ID, "[email protected]").await+}++async fn insert_team(pool: &PgPool, team_id: Uuid, owner_id: &str) -> anyhow::Result<()> {+ sqlx::query!(+ r#"INSERT INTO team (id, owner_id, name) VALUES ($1, $2, $3)"#,+ team_id,+ owner_id,+ team_id.to_string(),+ )+ .execute(pool)+ .await?;+ Ok(())+}++async fn insert_team_member(pool: &PgPool, user_id: &str, team_id: Uuid) -> anyhow::Result<()> {+ sqlx::query!(+ r#"INSERT INTO team_user (user_id, team_id, team_role) VALUES ($1, $2, 'member')"#,+ user_id,+ team_id+ )+ .execute(pool)+ .await?;+ Ok(())+}+async fn create_webhook(repo: &PgRepository) -> Webhook {repo.create_webhook(user_id(),@@ -363,6 +400,92 @@ async fn list_matching_event_excludes_ineligible_webhooks(pool: PgPool) -> anyhoOk(())}+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn resolved_workspaces_match_personal_and_related_team_webhooks(+ pool: PgPool,+) -> anyhow::Result<()> {+ insert_user(&pool).await?;+ insert_user_with_id(&pool, SECOND_USER_ID, "[email protected]").await?;++ let team_id = Uuid::parse_str(TEAM_ID)?;+ let unrelated_team_id = Uuid::parse_str(UNRELATED_TEAM_ID)?;+ insert_team(&pool, team_id, USER_ID).await?;+ insert_team(&pool, unrelated_team_id, SECOND_USER_ID).await?;+ insert_team_member(&pool, USER_ID, team_id).await?;+ insert_team_member(&pool, SECOND_USER_ID, team_id).await?;++ let matching_filter = vec![webhook_filter(&["document.created"], None)];+ insert_webhook_for_matching(+ &pool,+ "wh_personal",+ USER_ID,+ matching_filter.clone(),+ WebhookStatus::Active,+ true,+ false,+ )+ .await?;+ insert_webhook_for_matching(+ &pool,+ "wh_team",+ TEAM_ID,+ matching_filter.clone(),+ WebhookStatus::Active,+ true,+ false,+ )+ .await?;+ insert_webhook_for_matching(+ &pool,+ "wh_unrelated_team",+ UNRELATED_TEAM_ID,+ matching_filter,+ WebhookStatus::Active,+ true,+ false,+ )+ .await?;++ let repo = PgRepository::new(pool);+ let workspace_ids = repo+ .resolve_workspace_ids(vec![+ second_user_id(),+ user_id(),+ second_user_id(),+ user_id(),+ ])+ .await?;++ assert_eq!(+ workspace_ids,+ vec![+ TEAM_ID.to_string(),+ USER_ID.to_string(),+ SECOND_USER_ID.to_string(),+ ]+ );++ let webhooks = repo+ .list_active_webhooks_matching_event(+ workspace_ids,+ "document.created".to_string(),+ "doc_1".to_string(),+ )+ .await?;++ assert_eq!(webhook_ids(&webhooks), vec!["wh_personal", "wh_team"]);+ Ok(())+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn resolve_workspace_ids_returns_empty_for_no_people(pool: PgPool) -> anyhow::Result<()> {+ let repo = PgRepository::new(pool.clone());+ pool.close().await;++ assert!(repo.resolve_workspace_ids(Vec::new()).await?.is_empty());+ Ok(())+}+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]async fn patch_updates_endpoint_and_resets_validity(pool: PgPool) -> anyhow::Result<()> {insert_user(&pool).await?;@@ -431,21 +554,8 @@ async fn get_webhook_excludes_deleted_rows(pool: PgPool) -> anyhow::Result<()> {async fn get_user_team_workspace_id_returns_team_membership(pool: PgPool) -> anyhow::Result<()> {insert_user(&pool).await?;let team_id = Uuid::parse_str(TEAM_ID)?;- sqlx::query!(- r#"INSERT INTO team (id, owner_id, name) VALUES ($1, $2, $3)"#,- team_id,- USER_ID,- team_id.to_string(),- )- .execute(&pool)- .await?;- sqlx::query!(- r#"INSERT INTO team_user (user_id, team_id, team_role) VALUES ($1, $2, 'member')"#,- USER_ID,- team_id- )- .execute(&pool)- .await?;+ insert_team(&pool, team_id, USER_ID).await?;+ insert_team_member(&pool, USER_ID, team_id).await?;let repo = PgRepository::new(pool);assert_eq!(diff --git a/rust/cloud-storage/webhook/src/outbound/sqs_queue.rs b/rust/cloud-storage/webhook/src/outbound/sqs_queue.rsnew file mode 100644index 0000000000..b29888b53a--- /dev/null+++ b/rust/cloud-storage/webhook/src/outbound/sqs_queue.rs@@ -0,0 +1,148 @@+//! SQS-backed webhook event queue.++#[cfg(test)]+#[path = "sqs_queue_test.rs"]+mod sqs_queue_test;++use std::{sync::Arc, time::Duration};++use anyhow::{Context, ensure};+use sqs_client::{ReceivedMessage, SQS};++use crate::domain::{+ models::{RawWebhookEventQueueMessage, WebhookEventQueueMessage},+ ports::{WebhookEventEnqueuer, WebhookEventQueue},+};++const MAX_VISIBILITY_TIMEOUT_SECONDS: u64 = 12 * 60 * 60;++/// SQS-backed implementation of webhook enqueueing and worker queue operations.+#[derive(Clone)]+pub struct SqsWebhookQueue {+ sqs: Arc<SQS>,+ queue_url: String,+ max_messages: i32,+ wait_time_seconds: i32,+}++impl SqsWebhookQueue {+ /// Create a webhook queue adapter pointing at `queue_url`.+ pub fn new(+ sqs: Arc<SQS>,+ queue_url: String,+ max_messages: i32,+ wait_time_seconds: i32,+ ) -> Self {+ Self {+ sqs,+ queue_url,+ max_messages,+ wait_time_seconds,+ }+ }+}++impl WebhookEventEnqueuer for SqsWebhookQueue {+ type Err = anyhow::Error;++ #[tracing::instrument(+ err,+ skip(self, message),+ fields(+ webhook_id = %message.webhook_id,+ event_id = %message.event.event_id,+ )+ )]+ async fn enqueue(&self, message: WebhookEventQueueMessage) -> Result<(), Self::Err> {+ let prepared = prepare_webhook_message(&message)?;++ self.sqs+ .send_fifo_message(+ &self.queue_url,+ prepared.body,+ prepared.group_id,+ prepared.deduplication_id,+ )+ .await?;++ Ok(())+ }+}++impl WebhookEventQueue for SqsWebhookQueue {+ type Err = anyhow::Error;++ #[tracing::instrument(err, skip(self))]+ async fn receive_messages(&self) -> Result<Vec<RawWebhookEventQueueMessage>, Self::Err> {+ let messages = self+ .sqs+ .receive_messages(&self.queue_url, self.max_messages, self.wait_time_seconds)+ .await?;++ Ok(messages.into_iter().map(raw_webhook_message).collect())+ }++ #[tracing::instrument(err, skip(self, receipt_handle))]+ async fn delete_message(&self, receipt_handle: &str) -> Result<(), Self::Err> {+ self.sqs+ .delete_message(&self.queue_url, receipt_handle)+ .await?;++ Ok(())+ }++ #[tracing::instrument(+ err,+ skip(self, receipt_handle),+ fields(delay_seconds = delay.as_secs())+ )]+ async fn delay_message(&self, receipt_handle: &str, delay: Duration) -> Result<(), Self::Err> {+ let visibility_timeout = visibility_timeout_seconds(delay)?;++ self.sqs+ .change_message_visibility(&self.queue_url, receipt_handle, visibility_timeout)+ .await?;++ Ok(())+ }+}++#[derive(Debug, PartialEq, Eq)]+struct PreparedWebhookMessage {+ body: String,+ group_id: String,+ deduplication_id: String,+}++fn prepare_webhook_message(+ message: &WebhookEventQueueMessage,+) -> anyhow::Result<PreparedWebhookMessage> {+ let group_id = message.webhook_id.clone();+ let deduplication_id = format!("{}:{}", message.webhook_id, message.event.event_id);+ let body =+ serde_json::to_string(message).context("unable to serialize webhook queue message")?;++ Ok(PreparedWebhookMessage {+ body,+ group_id,+ deduplication_id,+ })+}++fn raw_webhook_message(message: ReceivedMessage) -> RawWebhookEventQueueMessage {+ RawWebhookEventQueueMessage {+ message_id: message.message_id,+ body: message.body,+ receipt_handle: message.receipt_handle,+ }+}++fn visibility_timeout_seconds(delay: Duration) -> anyhow::Result<i32> {+ let seconds = delay.as_secs();+ ensure!(+ seconds <= MAX_VISIBILITY_TIMEOUT_SECONDS,+ "webhook message visibility delay cannot exceed {MAX_VISIBILITY_TIMEOUT_SECONDS} seconds"+ );++ Ok(seconds as i32)+}diff --git a/rust/cloud-storage/webhook/src/outbound/sqs_queue_test.rs b/rust/cloud-storage/webhook/src/outbound/sqs_queue_test.rsnew file mode 100644index 0000000000..6a55411bee--- /dev/null+++ b/rust/cloud-storage/webhook/src/outbound/sqs_queue_test.rs@@ -0,0 +1,74 @@+use super::*;+use crate::domain::models::WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION;+use serde_json::json;++fn sample_message() -> WebhookEventQueueMessage {+ serde_json::from_value(json!({+ "version": WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION,+ "webhook_id": "wh_123",+ "event": {+ "event_id": "event_456",+ "schema_version": 1,+ "event_name": "document.created",+ "entity_type": "document",+ "entity_id": "doc_789",+ "ordering_key": "doc_789",+ "occurred_at": "2026-07-10T12:00:00Z",+ "broker_envelope": {+ "event_id": "event_456",+ "payload": { "document_id": "doc_789" }+ }+ }+ }))+ .unwrap()+}++#[test]+fn webhook_message_is_serialized_as_the_typed_queue_contract() {+ let message = sample_message();+ let prepared = prepare_webhook_message(&message).unwrap();+ let deserialized: WebhookEventQueueMessage = serde_json::from_str(&prepared.body).unwrap();++ assert_eq!(deserialized, message);+}++#[test]+fn webhook_message_uses_webhook_id_as_fifo_group_id() {+ let prepared = prepare_webhook_message(&sample_message()).unwrap();++ assert_eq!(prepared.group_id, "wh_123");+}++#[test]+fn webhook_message_combines_webhook_and_event_ids_for_fifo_deduplication() {+ let prepared = prepare_webhook_message(&sample_message()).unwrap();++ assert_eq!(prepared.deduplication_id, "wh_123:event_456");+}++#[test]+fn raw_webhook_message_retains_transport_fields() {+ let raw = raw_webhook_message(ReceivedMessage {+ message_id: Some("message-id".to_string()),+ body: Some("message-body".to_string()),+ receipt_handle: Some("receipt-handle".to_string()),+ });++ assert_eq!(raw.message_id.as_deref(), Some("message-id"));+ assert_eq!(raw.body.as_deref(), Some("message-body"));+ assert_eq!(raw.receipt_handle.as_deref(), Some("receipt-handle"));+}++#[test]+fn visibility_timeout_rejects_values_above_the_sqs_limit() {+ assert_eq!(+ visibility_timeout_seconds(Duration::from_secs(43_200)).unwrap(),+ 43_200,+ );+ assert_eq!(+ visibility_timeout_seconds(Duration::from_secs(43_201))+ .unwrap_err()+ .to_string(),+ "webhook message visibility delay cannot exceed 43200 seconds",+ );+}