Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(webhooks): process webhook consumer events and send items to webhooks (#4710)

Will Hutchinson74a026af7926
diff --git a/docker-compose.local-e2e.yml b/docker-compose.local-e2e.yml
index c415b5dfc7..481f82ebb7 100644
--- a/docker-compose.local-e2e.yml
+++ b/docker-compose.local-e2e.yml
@@ -43,6 +43,7 @@ x-local-e2e-env: &local-e2e-env
OVERRIDE_NOTIFICATION_QUEUE: http://localstack:4566/000000000000/notification-queue
OVERRIDE_NOTIFICATION_INGRESS_QUEUE: http://localstack:4566/000000000000/notification-ingress-queue
OVERRIDE_PUSH_NOTIFICATION_EVENT_HANDLER_QUEUE: push-delivery-queue
+ OVERRIDE_WEBHOOK_EVENT_QUEUE: http://localstack:4566/000000000000/webhook-event-queue.fifo
OVERRIDE_SEARCH_EVENT_QUEUE: search-event-queue
OVERRIDE_AI_PROJECTION_QUEUE: http://localstack:4566/000000000000/ai-projection-queue
OVERRIDE_SFS_DELETE_QUEUE: email-sfs-delete-queue
diff --git a/infra/stacks/cloud-storage-service/index.ts b/infra/stacks/cloud-storage-service/index.ts
index 12b134694a..bab7dcc820 100644
--- a/infra/stacks/cloud-storage-service/index.ts
+++ b/infra/stacks/cloud-storage-service/index.ts
@@ -1,6 +1,6 @@
import * as aws from '@pulumi/aws';
import * as pulumi from '@pulumi/pulumi';
-import { createBucket } from '../../packages/resources';
+import { createBucket, Queue } from '../../packages/resources';
import {
config,
getMacroApiToken,
@@ -30,6 +30,16 @@ const tags = {
project: 'cloud-storage-service',
};
+const webhookEventQueue = new Queue('webhook-event', {
+ fifoQueue: true,
+ maxReceiveCount: 20,
+ tags,
+ visibilityTimeoutSeconds: 30,
+});
+
+export const webhookEventQueueArn = webhookEventQueue.queue.arn;
+export const webhookEventQueueName = webhookEventQueue.queue.name;
+
const MACRO_CACHE = aws.secretsmanager
.getSecretVersionOutput({
secretId: config.require(`macro_cache_secret_key`),
@@ -246,6 +256,7 @@ const cloudStorageService = new CloudStorageService(
cloudStorageClusterName: cloudStorageClusterName,
queueArns: [
searchEventQueueArn,
+ webhookEventQueue.queue.arn,
deleteDocumentHandler.queue.arn,
notificationIngressQueueArn,
contactsQueueArn,
diff --git a/js/app/packages/service-clients/service-storage/generated/schemas/createWebhookRequest.ts b/js/app/packages/service-clients/service-storage/generated/schemas/createWebhookRequest.ts
index a6f79b6eb7..6070a15a3e 100644
--- a/js/app/packages/service-clients/service-storage/generated/schemas/createWebhookRequest.ts
+++ b/js/app/packages/service-clients/service-storage/generated/schemas/createWebhookRequest.ts
@@ -13,7 +13,7 @@ import type { WebhookScope } from './webhookScope';
* Request to create a webhook.
*/
export interface CreateWebhookRequest {
- /** HTTPS endpoint URL. */
+ /** Endpoint URL. HTTPS is required outside local environments. */
endpoint_url: string;
/** Typed filters used to match events and optional entity ids. */
filters: Vec;
diff --git a/js/app/packages/service-clients/service-storage/generated/schemas/patchWebhookRequest.ts b/js/app/packages/service-clients/service-storage/generated/schemas/patchWebhookRequest.ts
index d6fb1467e2..61f1555f2e 100644
--- a/js/app/packages/service-clients/service-storage/generated/schemas/patchWebhookRequest.ts
+++ b/js/app/packages/service-clients/service-storage/generated/schemas/patchWebhookRequest.ts
@@ -14,7 +14,7 @@ import type { PatchWebhookRequestStatus } from './patchWebhookRequestStatus';
* Request to patch a webhook.
*/
export interface PatchWebhookRequest {
- /** HTTPS endpoint URL. */
+ /** Endpoint URL. HTTPS is required outside local environments. */
endpoint_url?: PatchWebhookRequestEndpointUrl;
filters?: PatchWebhookRequestFilters;
headers?: PatchWebhookRequestHeaders;
diff --git a/js/app/packages/service-clients/service-storage/generated/schemas/patchWebhookRequestEndpointUrl.ts b/js/app/packages/service-clients/service-storage/generated/schemas/patchWebhookRequestEndpointUrl.ts
index 6aa42093a7..f6c13caa1e 100644
--- a/js/app/packages/service-clients/service-storage/generated/schemas/patchWebhookRequestEndpointUrl.ts
+++ b/js/app/packages/service-clients/service-storage/generated/schemas/patchWebhookRequestEndpointUrl.ts
@@ -6,6 +6,6 @@
*/
/**
- * HTTPS endpoint URL.
+ * Endpoint URL. HTTPS is required outside local environments.
*/
export type PatchWebhookRequestEndpointUrl = string | null;
diff --git a/js/app/packages/service-clients/service-storage/generated/schemas/webhook.ts b/js/app/packages/service-clients/service-storage/generated/schemas/webhook.ts
index 57bf5efc13..7b1f50072e 100644
--- a/js/app/packages/service-clients/service-storage/generated/schemas/webhook.ts
+++ b/js/app/packages/service-clients/service-storage/generated/schemas/webhook.ts
@@ -21,7 +21,7 @@ export interface Webhook {
created_by_user_id: string;
/** Soft-delete timestamp. */
deleted_at?: WebhookDeletedAt;
- /** HTTPS endpoint URL. */
+ /** Endpoint URL. HTTPS is required outside local environments. */
endpoint_url: string;
/** Typed filters used to match events and optional entity ids. */
filters: Vec;
diff --git a/js/app/packages/service-clients/service-storage/generated/zod.ts b/js/app/packages/service-clients/service-storage/generated/zod.ts
index 5703c264ba..7718dc353c 100644
--- a/js/app/packages/service-clients/service-storage/generated/zod.ts
+++ b/js/app/packages/service-clients/service-storage/generated/zod.ts
@@ -20029,7 +20029,9 @@ export const editProjectV2Response = zod.object({
*/
export const createWebhookBody = zod
.object({
- endpoint_url: zod.string().describe('HTTPS endpoint URL.'),
+ endpoint_url: zod
+ .string()
+ .describe('Endpoint URL. HTTPS is required outside local environments.'),
filters: zod.array(
zod
.object({
@@ -20073,7 +20075,10 @@ export const patchWebhookParams = zod.object({
export const patchWebhookBody = zod
.object({
- endpoint_url: zod.string().nullish().describe('HTTPS endpoint URL.'),
+ endpoint_url: zod
+ .string()
+ .nullish()
+ .describe('Endpoint URL. HTTPS is required outside local environments.'),
filters: zod
.union([
zod.null(),
@@ -20119,7 +20124,9 @@ export const patchWebhookResponse = zod
.datetime({})
.nullish()
.describe('Soft-delete timestamp.'),
- endpoint_url: zod.string().describe('HTTPS endpoint URL.'),
+ endpoint_url: zod
+ .string()
+ .describe('Endpoint URL. HTTPS is required outside local environments.'),
filters: zod.array(
zod
.object({
diff --git a/js/app/packages/service-clients/service-storage/openapi.json b/js/app/packages/service-clients/service-storage/openapi.json
index e3e10f62a6..3d329b1e4f 100644
--- a/js/app/packages/service-clients/service-storage/openapi.json
+++ b/js/app/packages/service-clients/service-storage/openapi.json
@@ -12627,7 +12627,7 @@
"properties": {
"endpoint_url": {
"type": "string",
- "description": "HTTPS endpoint URL."
+ "description": "Endpoint URL. HTTPS is required outside local environments."
},
"filters": {
"$ref": "#/components/schemas/Vec",
@@ -16341,7 +16341,7 @@
"properties": {
"endpoint_url": {
"type": ["string", "null"],
- "description": "HTTPS endpoint URL."
+ "description": "Endpoint URL. HTTPS is required outside local environments."
},
"filters": {
"oneOf": [
@@ -19830,7 +19830,7 @@
},
"endpoint_url": {
"type": "string",
- "description": "HTTPS endpoint URL."
+ "description": "Endpoint URL. HTTPS is required outside local environments."
},
"filters": {
"$ref": "#/components/schemas/Vec",
diff --git a/local_stack.just b/local_stack.just
index effc4afbb7..d80fa3612e 100644
--- a/local_stack.just
+++ b/local_stack.just
@@ -29,6 +29,7 @@ create_local_queues:
aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name notification-queue || true
aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name notification-ingress-queue || true
aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name push-delivery-queue || true
+ aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name webhook-event-queue.fifo --attributes FifoQueue=true || true
aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name email-service-backfill-queue || true
aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name delete-chat-handler-queue || true
aws --endpoint-url=http://localhost:4566 sqs create-queue --queue-name contacts-queue || true
diff --git a/nix/cloud-storage.nix b/nix/cloud-storage.nix
index 34c694bdef..cc2c9964d6 100644
--- a/nix/cloud-storage.nix
+++ b/nix/cloud-storage.nix
@@ -668,6 +668,7 @@
parallel
docker-compose
curl
+ kcat
xz
unzip
zip
diff --git a/rust/cloud-storage/.sqlx/query-101a8c8116e33aba8ce97f193b8008c5fc34bcf0a874b39ba89f0558771f78d2.json b/rust/cloud-storage/.sqlx/query-101a8c8116e33aba8ce97f193b8008c5fc34bcf0a874b39ba89f0558771f78d2.json
new file mode 100644
index 0000000000..d1480861e1
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-101a8c8116e33aba8ce97f193b8008c5fc34bcf0a874b39ba89f0558771f78d2.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE webhook_delivery\n SET\n status = $2,\n next_attempt_at = NULL,\n updated_at = now()\n WHERE id = $1\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "101a8c8116e33aba8ce97f193b8008c5fc34bcf0a874b39ba89f0558771f78d2"
+}
diff --git a/rust/cloud-storage/.sqlx/query-1b000e3e86b627dac7ffb345ddc56b1507959639505feedee009c2fa3cea8fdf.json b/rust/cloud-storage/.sqlx/query-1b000e3e86b627dac7ffb345ddc56b1507959639505feedee009c2fa3cea8fdf.json
new file mode 100644
index 0000000000..8738a45ea4
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-1b000e3e86b627dac7ffb345ddc56b1507959639505feedee009c2fa3cea8fdf.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE webhook\n SET\n last_success_at = GREATEST(COALESCE(last_success_at, $2), $2),\n updated_at = GREATEST(updated_at, $2)\n WHERE id = $1\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Timestamptz"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "1b000e3e86b627dac7ffb345ddc56b1507959639505feedee009c2fa3cea8fdf"
+}
diff --git a/rust/cloud-storage/.sqlx/query-35cbfa36e3f14269f1bb5e18ab3a1d67a27f633f50736f8e92c59344b209b801.json b/rust/cloud-storage/.sqlx/query-35cbfa36e3f14269f1bb5e18ab3a1d67a27f633f50736f8e92c59344b209b801.json
new file mode 100644
index 0000000000..77f31f2e3f
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-35cbfa36e3f14269f1bb5e18ab3a1d67a27f633f50736f8e92c59344b209b801.json
@@ -0,0 +1,119 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT\n d.id AS delivery_id,\n d.status AS delivery_status,\n d.attempt_count,\n d.next_attempt_at,\n w.id AS webhook_id,\n w.workspace_id,\n w.name,\n w.endpoint_url,\n w.signing_secret,\n w.headers,\n w.status AS webhook_status,\n w.is_valid,\n w.created_by_user_id,\n w.created_at,\n w.updated_at,\n w.deleted_at,\n w.filters\n FROM webhook_delivery d\n JOIN webhook w ON w.id = d.webhook_id\n WHERE d.webhook_id = $1\n AND d.event_id = $2\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "delivery_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 1,
+ "name": "delivery_status",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 2,
+ "name": "attempt_count",
+ "type_info": "Int4"
+ },
+ {
+ "ordinal": 3,
+ "name": "next_attempt_at",
+ "type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 4,
+ "name": "webhook_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 5,
+ "name": "workspace_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 6,
+ "name": "name",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 7,
+ "name": "endpoint_url",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 8,
+ "name": "signing_secret",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 9,
+ "name": "headers",
+ "type_info": "Jsonb"
+ },
+ {
+ "ordinal": 10,
+ "name": "webhook_status",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 11,
+ "name": "is_valid",
+ "type_info": "Bool"
+ },
+ {
+ "ordinal": 12,
+ "name": "created_by_user_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 13,
+ "name": "created_at",
+ "type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 14,
+ "name": "updated_at",
+ "type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 15,
+ "name": "deleted_at",
+ "type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 16,
+ "name": "filters",
+ "type_info": "Jsonb"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ false,
+ true,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ false,
+ true,
+ false
+ ]
+ },
+ "hash": "35cbfa36e3f14269f1bb5e18ab3a1d67a27f633f50736f8e92c59344b209b801"
+}
diff --git a/rust/cloud-storage/.sqlx/query-360592f66d7d22bee5c37c9c1a614b382288e75c366ffe18bd663560cb250933.json b/rust/cloud-storage/.sqlx/query-360592f66d7d22bee5c37c9c1a614b382288e75c366ffe18bd663560cb250933.json
new file mode 100644
index 0000000000..b5b96b9413
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-360592f66d7d22bee5c37c9c1a614b382288e75c366ffe18bd663560cb250933.json
@@ -0,0 +1,20 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE webhook_delivery\n SET\n status = $2,\n next_attempt_at = $3,\n delivered_at = $4,\n updated_at = $5\n WHERE id = $1\n AND status = $6\n AND attempt_count = $7\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text",
+ "Timestamptz",
+ "Timestamptz",
+ "Timestamptz",
+ "Text",
+ "Int4"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "360592f66d7d22bee5c37c9c1a614b382288e75c366ffe18bd663560cb250933"
+}
diff --git a/rust/cloud-storage/.sqlx/query-366018afc77416e3ef92d6e46f9f1f6f44c74c8461c3110f074a09abc2983a72.json b/rust/cloud-storage/.sqlx/query-366018afc77416e3ef92d6e46f9f1f6f44c74c8461c3110f074a09abc2983a72.json
new file mode 100644
index 0000000000..11a0712f24
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-366018afc77416e3ef92d6e46f9f1f6f44c74c8461c3110f074a09abc2983a72.json
@@ -0,0 +1,32 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE webhook_delivery_attempt\n SET\n status = $4,\n completed_at = now(),\n duration_ms = $5,\n response_status = $6,\n response_headers_redacted = $7,\n response_body_preview = $8,\n error_kind = $9,\n error_message = $10\n WHERE id = $1\n AND webhook_delivery_id = $2\n AND attempt_number = $3\n AND status = $11\n RETURNING completed_at AS \"completed_at!\"\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "completed_at!",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text",
+ "Int4",
+ "Text",
+ "Int4",
+ "Int4",
+ "Jsonb",
+ "Text",
+ "Text",
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": [
+ true
+ ]
+ },
+ "hash": "366018afc77416e3ef92d6e46f9f1f6f44c74c8461c3110f074a09abc2983a72"
+}
diff --git a/rust/cloud-storage/.sqlx/query-373397112fdf355b6fa294d50eee28cae14ac674f49c11349823e4b3c2e13938.json b/rust/cloud-storage/.sqlx/query-373397112fdf355b6fa294d50eee28cae14ac674f49c11349823e4b3c2e13938.json
new file mode 100644
index 0000000000..df24ceafb1
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-373397112fdf355b6fa294d50eee28cae14ac674f49c11349823e4b3c2e13938.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE webhook\n SET\n last_failure_at = GREATEST(COALESCE(last_failure_at, $2), $2),\n updated_at = GREATEST(updated_at, $2)\n WHERE id = $1\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Timestamptz"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "373397112fdf355b6fa294d50eee28cae14ac674f49c11349823e4b3c2e13938"
+}
diff --git a/rust/cloud-storage/.sqlx/query-383d8a8ec9c041983d350fda064aefb8d058d7dc39b585bbc9c52070b4eed94f.json b/rust/cloud-storage/.sqlx/query-383d8a8ec9c041983d350fda064aefb8d058d7dc39b585bbc9c52070b4eed94f.json
new file mode 100644
index 0000000000..f076cc9654
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-383d8a8ec9c041983d350fda064aefb8d058d7dc39b585bbc9c52070b4eed94f.json
@@ -0,0 +1,29 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE webhook_delivery\n SET\n status = $2,\n attempt_count = attempt_count + 1,\n first_attempt_at = COALESCE(first_attempt_at, now()),\n last_attempt_at = now(),\n next_attempt_at = NULL,\n updated_at = now()\n WHERE id = $1\n RETURNING\n attempt_count,\n last_attempt_at AS \"started_at!\"\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "attempt_count",
+ "type_info": "Int4"
+ },
+ {
+ "ordinal": 1,
+ "name": "started_at!",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": [
+ false,
+ true
+ ]
+ },
+ "hash": "383d8a8ec9c041983d350fda064aefb8d058d7dc39b585bbc9c52070b4eed94f"
+}
diff --git a/rust/cloud-storage/.sqlx/query-5dd893b03be604e8ac4d82f90e8f205ea8a52b85cedfb30a314c1b8f891d25ad.json b/rust/cloud-storage/.sqlx/query-5dd893b03be604e8ac4d82f90e8f205ea8a52b85cedfb30a314c1b8f891d25ad.json
new file mode 100644
index 0000000000..8ac25d6415
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-5dd893b03be604e8ac4d82f90e8f205ea8a52b85cedfb30a314c1b8f891d25ad.json
@@ -0,0 +1,40 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT\n webhook_id,\n status,\n attempt_count,\n COALESCE(next_attempt_at <= now(), true) AS \"is_due!\"\n FROM webhook_delivery\n WHERE id = $1\n FOR UPDATE\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "webhook_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 1,
+ "name": "status",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 2,
+ "name": "attempt_count",
+ "type_info": "Int4"
+ },
+ {
+ "ordinal": 3,
+ "name": "is_due!",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ false,
+ null
+ ]
+ },
+ "hash": "5dd893b03be604e8ac4d82f90e8f205ea8a52b85cedfb30a314c1b8f891d25ad"
+}
diff --git a/rust/cloud-storage/.sqlx/query-5e76ca3bdaae6433ae6091b20624df7c65298da23845b70ed936814b82591dfc.json b/rust/cloud-storage/.sqlx/query-5e76ca3bdaae6433ae6091b20624df7c65298da23845b70ed936814b82591dfc.json
new file mode 100644
index 0000000000..6eff6ab251
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-5e76ca3bdaae6433ae6091b20624df7c65298da23845b70ed936814b82591dfc.json
@@ -0,0 +1,34 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT webhook_id, status, attempt_count\n FROM webhook_delivery\n WHERE id = $1\n FOR UPDATE\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "webhook_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 1,
+ "name": "status",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 2,
+ "name": "attempt_count",
+ "type_info": "Int4"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ false
+ ]
+ },
+ "hash": "5e76ca3bdaae6433ae6091b20624df7c65298da23845b70ed936814b82591dfc"
+}
diff --git a/rust/cloud-storage/.sqlx/query-66c1f8f142ef853861e45f6292689cab233fad58563c99c4f5b349f9b02bc508.json b/rust/cloud-storage/.sqlx/query-66c1f8f142ef853861e45f6292689cab233fad58563c99c4f5b349f9b02bc508.json
new file mode 100644
index 0000000000..156c4c0178
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-66c1f8f142ef853861e45f6292689cab233fad58563c99c4f5b349f9b02bc508.json
@@ -0,0 +1,18 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO webhook_delivery_attempt (\n id,\n webhook_delivery_id,\n attempt_number,\n status,\n started_at\n )\n VALUES ($1, $2, $3, $4, $5)\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text",
+ "Int4",
+ "Text",
+ "Timestamptz"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "66c1f8f142ef853861e45f6292689cab233fad58563c99c4f5b349f9b02bc508"
+}
diff --git a/rust/cloud-storage/.sqlx/query-683604acb3fb29571b0f79a5bba4fa82f3e2ba9358d051a4249738032f49f06f.json b/rust/cloud-storage/.sqlx/query-683604acb3fb29571b0f79a5bba4fa82f3e2ba9358d051a4249738032f49f06f.json
new file mode 100644
index 0000000000..65c70b1bea
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-683604acb3fb29571b0f79a5bba4fa82f3e2ba9358d051a4249738032f49f06f.json
@@ -0,0 +1,25 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE webhook_delivery_attempt\n SET\n status = $3,\n completed_at = now()\n WHERE webhook_delivery_id = $1\n AND attempt_number = $2\n AND status = $4\n RETURNING completed_at AS \"completed_at!\"\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "completed_at!",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Int4",
+ "Text",
+ "Text"
+ ]
+ },
+ "nullable": [
+ true
+ ]
+ },
+ "hash": "683604acb3fb29571b0f79a5bba4fa82f3e2ba9358d051a4249738032f49f06f"
+}
diff --git a/rust/cloud-storage/.sqlx/query-7e0d5cd72591beaac66052550a260f7ac3f98e909a1ed7eeea9d321cc600e528.json b/rust/cloud-storage/.sqlx/query-7e0d5cd72591beaac66052550a260f7ac3f98e909a1ed7eeea9d321cc600e528.json
new file mode 100644
index 0000000000..c95a9be0e4
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-7e0d5cd72591beaac66052550a260f7ac3f98e909a1ed7eeea9d321cc600e528.json
@@ -0,0 +1,34 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT webhook_id, status, attempt_count\n FROM webhook_delivery\n WHERE id = $1\n FOR UPDATE\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "webhook_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 1,
+ "name": "status",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 2,
+ "name": "attempt_count",
+ "type_info": "Int4"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Text"
+ ]
+ },
+ "nullable": [
+ false,
+ false,
+ false
+ ]
+ },
+ "hash": "7e0d5cd72591beaac66052550a260f7ac3f98e909a1ed7eeea9d321cc600e528"
+}
diff --git a/rust/cloud-storage/.sqlx/query-9fa8e173d6b50e682450e57c1b5021738e91c289eb7cb1cf26c6637ac9d2ef60.json b/rust/cloud-storage/.sqlx/query-9fa8e173d6b50e682450e57c1b5021738e91c289eb7cb1cf26c6637ac9d2ef60.json
new file mode 100644
index 0000000000..b4320af041
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-9fa8e173d6b50e682450e57c1b5021738e91c289eb7cb1cf26c6637ac9d2ef60.json
@@ -0,0 +1,25 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO webhook_delivery (\n id,\n webhook_id,\n event_id,\n event,\n event_schema_version,\n event_occurred_at,\n event_entity_type,\n event_entity_id,\n event_ordering_key,\n status,\n request_url,\n request_headers_redacted,\n request_body\n )\n SELECT\n $1,\n w.id,\n $3,\n $4,\n $5,\n $6,\n $7,\n $8,\n $9,\n $10,\n w.endpoint_url,\n COALESCE(\n (\n SELECT jsonb_object_agg(header_name, $11::TEXT)\n FROM jsonb_object_keys(w.headers) AS header_name\n ),\n '{}'::JSONB\n ),\n $12\n FROM webhook w\n WHERE w.id = $2\n ON CONFLICT (webhook_id, event_id) DO NOTHING\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "Int4",
+ "Timestamptz",
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "Text",
+ "Jsonb"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "9fa8e173d6b50e682450e57c1b5021738e91c289eb7cb1cf26c6637ac9d2ef60"
+}
diff --git a/rust/cloud-storage/.sqlx/query-f3219580edf757829eb259ed116238fa1f567fab31febb2c7f7400dfcbdb6608.json b/rust/cloud-storage/.sqlx/query-f3219580edf757829eb259ed116238fa1f567fab31febb2c7f7400dfcbdb6608.json
new file mode 100644
index 0000000000..8acf4324fa
--- /dev/null
+++ b/rust/cloud-storage/.sqlx/query-f3219580edf757829eb259ed116238fa1f567fab31febb2c7f7400dfcbdb6608.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT DISTINCT team_id::text AS \"team_id!\"\n FROM team_user\n WHERE user_id = ANY($1)\n ORDER BY team_id::text\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "team_id!",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "TextArray"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "f3219580edf757829eb259ed116238fa1f567fab31febb2c7f7400dfcbdb6608"
+}
diff --git a/rust/cloud-storage/Cargo.lock b/rust/cloud-storage/Cargo.lock
index 160afff90c..54b7e8d819 100644
--- a/rust/cloud-storage/Cargo.lock
+++ b/rust/cloud-storage/Cargo.lock
@@ -14870,6 +14870,7 @@ dependencies = [
"chrono",
"documents",
"entity_access",
+ "futures",
"hex",
"hmac 0.12.1",
"macro_db_migrator",
@@ -14888,6 +14889,7 @@ dependencies = [
"serde_json",
"sha2 0.10.9",
"sqlx",
+ "sqs_client",
"thiserror 2.0.18",
"tokio",
"tower 0.5.3",
diff --git a/rust/cloud-storage/document_storage_service/Cargo.toml b/rust/cloud-storage/document_storage_service/Cargo.toml
index 3afb1df745..1c15f12328 100644
--- a/rust/cloud-storage/document_storage_service/Cargo.toml
+++ b/rust/cloud-storage/document_storage_service/Cargo.toml
@@ -163,7 +163,7 @@ utoipa = { workspace = true, features = ["axum_extras"] }
utoipa-swagger-ui = { workspace = true, features = ["axum"] }
uuid = { workspace = true }
zip = { workspace = true }
-webhook = { path = "../webhook", features = ["inbound", "outbound"] }
+webhook = { path = "../webhook", features = ["inbound", "outbound", "worker"] }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
macro_config = { path = "../macro_config" }
doppler-config = { workspace = true }
diff --git a/rust/cloud-storage/document_storage_service/src/config.rs b/rust/cloud-storage/document_storage_service/src/config.rs
index d878e39c7c..a13f03b471 100644
--- a/rust/cloud-storage/document_storage_service/src/config.rs
+++ b/rust/cloud-storage/document_storage_service/src/config.rs
@@ -125,6 +125,13 @@ pub struct Config {
#[macro_config_default(4)]
pub queue_wait_time_seconds: i32,
+ /// Maximum number of SQS messages to receive per webhook worker poll
+ #[macro_config_default(10)]
+ pub webhook_queue_max_messages: i32,
+ /// SQS long-poll wait time in seconds for the webhook worker
+ #[macro_config_default(20)]
+ pub webhook_queue_wait_time_seconds: i32,
+
/// The document limit for free users
#[macro_config_default(20)]
pub document_limit: u64,
diff --git a/rust/cloud-storage/document_storage_service/src/main.rs b/rust/cloud-storage/document_storage_service/src/main.rs
index 4fd027c02e..0105cfd899 100644
--- a/rust/cloud-storage/document_storage_service/src/main.rs
+++ b/rust/cloud-storage/document_storage_service/src/main.rs
@@ -188,6 +188,12 @@ async fn main() -> anyhow::Result<()> {
let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(&aws_config))
.search_event_queue(&search_event_queue)
.document_delete_queue(&document_delete_queue);
+ let webhook_event_queue = webhook::outbound::SqsWebhookQueue::new(
+ Arc::new(sqs_client.clone()),
+ macro_queues::WebhookEventQueue::new().to_string(),
+ config.webhook_queue_max_messages,
+ config.webhook_queue_wait_time_seconds,
+ );
let contacts_ingress = Arc::new(contacts::domain::service::SqsContactsIngress {
queue: contacts::outbound::ingress::SqsContactsQueue::new(
@@ -575,11 +581,24 @@ async fn main() -> anyhow::Result<()> {
let call_state = CallRouterState::new(call_service.clone(), entity_access_service.clone());
let call_webhook_state = WebhookRouterState::new(call_service.clone());
- let webhook_service = webhook::domain::service::WebhookServiceImpl::new(
- webhook::outbound::pg_repository::PgRepository::new(db.clone()),
- webhook::outbound::http_validator::ReqwestWebhookValidationClient::new()
- .context("failed to create webhook validation client")?,
- );
+
+ let webhook_repository = webhook::outbound::PgRepository::new(db.clone());
+ let webhook_endpoint_scheme_policy = if matches!(env, Environment::Local) {
+ webhook::domain::models::WebhookEndpointSchemePolicy::HttpAndHttps
+ } else {
+ webhook::domain::models::WebhookEndpointSchemePolicy::HttpsOnly
+ };
+ let webhook_http_client =
+ webhook::outbound::ReqwestWebhookDeliveryClient::new_with_endpoint_scheme_policy(
+ webhook_endpoint_scheme_policy,
+ )
+ .context("failed to create webhook HTTP client")?;
+ let webhook_service =
+ webhook::domain::service::WebhookServiceImpl::new_with_endpoint_scheme_policy(
+ webhook_repository.clone(),
+ webhook_http_client.clone(),
+ webhook_endpoint_scheme_policy,
+ );
let webhook_rate_limiter = RateLimitServiceImpl {
repo: RedisRateLimitAdapter {
redis: redis_client.clone(),
@@ -590,23 +609,50 @@ async fn main() -> anyhow::Result<()> {
webhook_rate_limiter,
);
- // Webhook event ingestion: consume document and channel broker events and
- // fan them out to matching webhooks (per-event handlers are stubs for now).
let webhook_ingestion_service =
webhook::domain::ingestion::WebhookEventIngestionServiceImpl::new(
entity_access_service.clone(),
+ webhook_repository,
+ webhook_event_queue.clone(),
);
+ let webhook_delivery_repository =
+ webhook::outbound::PgWebhookDeliveryRepository::new(db.clone());
+ let webhook_delivery_service = webhook::domain::delivery::WebhookEventDeliveryServiceImpl::new(
+ webhook_delivery_repository,
+ webhook_http_client,
+ );
+ let webhook_worker = webhook::inbound::worker::WebhookEventWorker::new(
+ webhook_event_queue,
+ webhook_delivery_service,
+ );
+ let webhook_worker_task = tokio::spawn(async move {
+ tracing::info!("starting webhook event worker");
+ webhook_worker.run().await;
+ });
+ tokio::spawn(async move {
+ match webhook_worker_task.await {
+ Ok(()) => tracing::error!("webhook event worker exited unexpectedly"),
+ Err(error) => {
+ tracing::error!(error = ?error, "webhook event worker exited unexpectedly");
+ }
+ }
+ });
+
let webhook_consumer_brokers = config.kafka_brokers.as_ref().to_string();
tokio::spawn(async move {
loop {
- if let Err(e) = webhook::inbound::kafka_consumer::run_webhook_event_consumer(
+ tracing::info!("starting webhook event consumer");
+ let result = webhook::inbound::kafka_consumer::run_webhook_event_consumer(
&webhook_consumer_brokers,
webhook_ingestion_service.clone(),
std::future::pending::<()>(),
)
- .await
- {
- tracing::error!(error = ?e, "webhook event consumer exited");
+ .await;
+ match result {
+ Ok(()) => tracing::error!("webhook event consumer exited unexpectedly"),
+ Err(error) => {
+ tracing::error!(error = ?error, "webhook event consumer exited unexpectedly");
+ }
}
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
}
diff --git a/rust/cloud-storage/macro_queues/src/lib.rs b/rust/cloud-storage/macro_queues/src/lib.rs
index b44a7cc96b..3abe0334b4 100644
--- a/rust/cloud-storage/macro_queues/src/lib.rs
+++ b/rust/cloud-storage/macro_queues/src/lib.rs
@@ -540,6 +540,12 @@ queue! {
dev: "push-delivery-queue-dev",
prod: "push-delivery-queue-prod",
},
+ /// FIFO queue for webhook event delivery.
+ pub WebhookEventQueue {
+ local: "webhook-event-queue.fifo",
+ dev: "webhook-event-queue-dev.fifo",
+ prod: "webhook-event-queue-prod.fifo",
+ },
/// Queue for static-file-service S3 upload event notifications.
///
/// Note: this value is consumed as a full SQS URL in deployed
diff --git a/rust/cloud-storage/macro_queues/src/test.rs b/rust/cloud-storage/macro_queues/src/test.rs
index 1b04bcebd7..fbd88eadb8 100644
--- a/rust/cloud-storage/macro_queues/src/test.rs
+++ b/rust/cloud-storage/macro_queues/src/test.rs
@@ -129,6 +129,26 @@ fn grouped_defaults_do_not_check_overrides() {
);
}
+#[test]
+fn webhook_event_queue_defaults_are_fifo() {
+ assert_eq!(
+ WebhookEventQueue::local().as_ref(),
+ "webhook-event-queue.fifo"
+ );
+ assert_eq!(
+ WebhookEventQueue::dev().as_ref(),
+ "webhook-event-queue-dev.fifo",
+ );
+ assert_eq!(
+ WebhookEventQueue::prod().as_ref(),
+ "webhook-event-queue-prod.fifo",
+ );
+ assert_eq!(
+ WebhookEventQueue::local().override_env_var_name(),
+ "OVERRIDE_WEBHOOK_EVENT_QUEUE",
+ );
+}
+
#[test]
fn exported_queues_match_local_values() {
let queues = Queues::default_for_environment(Environment::Local);
@@ -175,6 +195,10 @@ fn exported_queues_match_local_values() {
queues.push_notification_event_handler_queue.as_ref(),
"push-delivery-queue",
);
+ assert_eq!(
+ queues.webhook_event_queue.as_ref(),
+ "webhook-event-queue.fifo",
+ );
assert_eq!(
queues.static_file_service_s3_event_queue_url.as_ref(),
"static-file-s3-event-notification-queue",
@@ -254,6 +278,10 @@ fn exported_queues_match_dev_values() {
queues.push_notification_event_handler_queue.as_ref(),
"push-delivery-queue-dev",
);
+ assert_eq!(
+ queues.webhook_event_queue.as_ref(),
+ "webhook-event-queue-dev.fifo",
+ );
assert_eq!(
queues.static_file_service_s3_event_queue_url.as_ref(),
"static-file-s3-event-notification-queue-dev",
@@ -342,6 +370,10 @@ fn exported_queues_match_prod_values() {
queues.push_notification_event_handler_queue.as_ref(),
"push-delivery-queue-prod",
);
+ assert_eq!(
+ queues.webhook_event_queue.as_ref(),
+ "webhook-event-queue-prod.fifo",
+ );
assert_eq!(
queues.static_file_service_s3_event_queue_url.as_ref(),
"static-file-s3-event-notification-queue-prod",
@@ -434,6 +466,10 @@ fn exported_queue_override_names_are_derived_from_env_var_names() {
PushNotificationEventHandlerQueue::local().override_env_var_name(),
"OVERRIDE_PUSH_NOTIFICATION_EVENT_HANDLER_QUEUE",
);
+ assert_eq!(
+ WebhookEventQueue::local().override_env_var_name(),
+ "OVERRIDE_WEBHOOK_EVENT_QUEUE",
+ );
// Pins the digit-boundary case: `S3` stays fused with a separator after it.
assert_eq!(
StaticFileServiceS3EventQueueUrl::local().override_env_var_name(),
diff --git a/rust/cloud-storage/sqs_client/src/lib.rs b/rust/cloud-storage/sqs_client/src/lib.rs
index 8b05979497..3ca831b2bb 100644
--- a/rust/cloud-storage/sqs_client/src/lib.rs
+++ b/rust/cloud-storage/sqs_client/src/lib.rs
@@ -1,4 +1,7 @@
mod message_attribute;
+mod queue;
+
+pub use queue::ReceivedMessage;
#[cfg(feature = "ai_projection")]
pub mod ai_projection;
diff --git a/rust/cloud-storage/sqs_client/src/queue.rs b/rust/cloud-storage/sqs_client/src/queue.rs
new file mode 100644
index 0000000000..80fbbe3699
--- /dev/null
+++ b/rust/cloud-storage/sqs_client/src/queue.rs
@@ -0,0 +1,98 @@
+//! Generic SQS transport operations shared by queue-specific adapters.
+
+use crate::SQS;
+
+/// Raw message returned from an SQS receive operation.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct ReceivedMessage {
+ /// Queue-assigned message identifier, when present.
+ pub message_id: Option<String>,
+ /// Unparsed message body, when present.
+ pub body: Option<String>,
+ /// Receipt handle used to delete or delay the message, when present.
+ pub receipt_handle: Option<String>,
+}
+
+impl SQS {
+ /// Send one message to a FIFO queue.
+ pub async fn send_fifo_message(
+ &self,
+ queue_url: &str,
+ message_body: String,
+ message_group_id: String,
+ message_deduplication_id: String,
+ ) -> anyhow::Result<()> {
+ self.inner
+ .send_message()
+ .queue_url(queue_url)
+ .message_body(message_body)
+ .message_group_id(message_group_id)
+ .message_deduplication_id(message_deduplication_id)
+ .send()
+ .await?;
+
+ Ok(())
+ }
+
+ /// Receive a batch of unparsed messages from a queue.
+ pub async fn receive_messages(
+ &self,
+ queue_url: &str,
+ max_number_of_messages: i32,
+ wait_time_seconds: i32,
+ ) -> anyhow::Result<Vec<ReceivedMessage>> {
+ let output = self
+ .inner
+ .receive_message()
+ .queue_url(queue_url)
+ .max_number_of_messages(max_number_of_messages)
+ .wait_time_seconds(wait_time_seconds)
+ .send()
+ .await?;
+
+ Ok(output
+ .messages
+ .unwrap_or_default()
+ .into_iter()
+ .map(|message| ReceivedMessage {
+ message_id: message.message_id,
+ body: message.body,
+ receipt_handle: message.receipt_handle,
+ })
+ .collect())
+ }
+
+ /// Delete a message from a queue using its receipt handle.
+ pub async fn delete_message(
+ &self,
+ queue_url: &str,
+ receipt_handle: &str,
+ ) -> anyhow::Result<()> {
+ self.inner
+ .delete_message()
+ .queue_url(queue_url)
+ .receipt_handle(receipt_handle)
+ .send()
+ .await?;
+
+ Ok(())
+ }
+
+ /// Change the visibility timeout for a received message.
+ pub async fn change_message_visibility(
+ &self,
+ queue_url: &str,
+ receipt_handle: &str,
+ visibility_timeout: i32,
+ ) -> anyhow::Result<()> {
+ self.inner
+ .change_message_visibility()
+ .queue_url(queue_url)
+ .receipt_handle(receipt_handle)
+ .visibility_timeout(visibility_timeout)
+ .send()
+ .await?;
+
+ Ok(())
+ }
+}
diff --git a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/local_env/test.rs b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/local_env/test.rs
index 16482f4bab..98d254afa6 100644
--- a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/local_env/test.rs
+++ b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/local_env/test.rs
@@ -53,6 +53,16 @@ fn values_are_local_only() {
}
}
+#[test]
+fn emits_webhook_fifo_queue_override_url() {
+ let env = local_env();
+ assert_eq!(
+ env.get(macro_queues::WebhookEventQueue::OVERRIDE_ENV_VAR_NAME)
+ .map(String::as_str),
+ Some("http://localstack:4566/000000000000/webhook-event-queue.fifo")
+ );
+}
+
#[test]
fn aws_creds_are_dummy() {
let env = local_env();
diff --git a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/localstack.rs b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/localstack.rs
index 54cce20c2c..f3ba39b54a 100644
--- a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/localstack.rs
+++ b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/localstack.rs
@@ -49,10 +49,11 @@ async fn provision_async(url: &str) -> Result<()> {
async fn create_queues(sqs: &aws_sdk_sqs::Client) -> Result<()> {
for queue in resources::QUEUES {
let name = queue.name;
- ignore_exists(
- sqs.create_queue().queue_name(name).send().await.map(|_| ()),
- &format!("queue {name}"),
- )?;
+ let mut request = sqs.create_queue().queue_name(name);
+ if name.ends_with(".fifo") {
+ request = request.attributes(aws_sdk_sqs::types::QueueAttributeName::FifoQueue, "true");
+ }
+ ignore_exists(request.send().await.map(|_| ()), &format!("queue {name}"))?;
}
Ok(())
}
diff --git a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/resources.rs b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/resources.rs
index 9f96de1175..28886a2aa2 100644
--- a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/resources.rs
+++ b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/resources.rs
@@ -106,6 +106,10 @@ pub const QUEUES: &[Queue] = &[
name: macro_queues::PushNotificationEventHandlerQueue::LOCAL,
bindings: &[("PUSH_NOTIFICATION_EVENT_HANDLER_QUEUE", Name)],
},
+ Queue {
+ name: macro_queues::WebhookEventQueue::LOCAL,
+ bindings: &[(macro_queues::WebhookEventQueue::OVERRIDE_ENV_VAR_NAME, Url)],
+ },
Queue {
name: macro_queues::EmailBackfillQueue::LOCAL,
bindings: &[("BACKFILL_QUEUE", Name), ("EMAIL_BACKFILL_QUEUE", Url)],
diff --git a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/resources/test.rs b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/resources/test.rs
index 38493fedf8..b58711add5 100644
--- a/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/resources/test.rs
+++ b/rust/cloud-storage/tools/xtask/crates/xtask_local/src/local/resources/test.rs
@@ -47,3 +47,23 @@ fn every_queue_is_bound() {
);
}
}
+
+#[test]
+fn webhook_fifo_queue_uses_a_full_url_override() {
+ let queue = QUEUES
+ .iter()
+ .find(|queue| queue.name == macro_queues::WebhookEventQueue::LOCAL)
+ .expect("webhook event queue is cataloged");
+
+ assert!(queue.name.ends_with(".fifo"));
+ assert_eq!(queue.bindings.len(), 1);
+ assert_eq!(
+ queue.bindings[0].0,
+ macro_queues::WebhookEventQueue::OVERRIDE_ENV_VAR_NAME
+ );
+ assert!(matches!(queue.bindings[0].1, QueueForm::Url));
+ assert_eq!(
+ queue.bindings[0].1.value(queue.name),
+ "http://localstack:4566/000000000000/webhook-event-queue.fifo"
+ );
+}
diff --git a/rust/cloud-storage/webhook/Cargo.toml b/rust/cloud-storage/webhook/Cargo.toml
index f7e2ca3f72..0765ffb539 100644
--- a/rust/cloud-storage/webhook/Cargo.toml
+++ b/rust/cloud-storage/webhook/Cargo.toml
@@ -29,6 +29,7 @@ ingestion = [
"dep:channels",
"dep:documents",
"dep:entity_access",
+ "dep:futures",
"dep:macro_event_broker",
"dep:uuid",
"ports",
@@ -40,11 +41,13 @@ outbound = [
"dep:sha2",
"dep:macro_uuid",
"dep:sqlx",
+ "dep:sqs_client",
"dep:tokio",
"dep:url",
"ports",
]
ports = ["dep:serde_json", "dep:url"]
+worker = ["dep:tokio", "ports"]
[dependencies]
anyhow = { workspace = true }
@@ -56,6 +59,7 @@ documents = { path = "../documents", default-features = false, optional = true }
entity_access = { path = "../entity_access", default-features = false, features = [
"ports",
], optional = true }
+futures = { workspace = true, optional = true }
hex = { workspace = true, optional = true }
hmac = { workspace = true, optional = true }
macro_env = { path = "../macro_env", optional = true }
@@ -72,6 +76,7 @@ serde = { workspace = true }
serde_json = { workspace = true, optional = true }
sha2 = { workspace = true, optional = true }
sqlx = { workspace = true, optional = true }
+sqs_client = { path = "../sqs_client", default-features = false, optional = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["macros", "time"], optional = true }
tracing = { workspace = true }
diff --git a/rust/cloud-storage/webhook/README.md b/rust/cloud-storage/webhook/README.md
index 394c74abd5..91d88f2830 100644
--- a/rust/cloud-storage/webhook/README.md
+++ b/rust/cloud-storage/webhook/README.md
@@ -1,58 +1,247 @@
# Webhook crate
-This crate currently contains the core application-layer webhook pieces for the first iteration.
+This crate provides webhook management, Kafka event ingestion, workspace and
+filter matching, FIFO SQS fan-out, tracked HTTP delivery, and retry handling.
+`document_storage_service` composes the production adapters and runs both the
+Kafka consumer and the queue worker.
-## Current scope
+## Management and validation
-- Database tables are present for event ingestion, webhooks, deliveries, and delivery attempts.
-- The Postgres repository supports creating, loading, patching, and updating webhook validity through the `is_valid` field. It also lists active webhooks whose typed `filters` match an event.
-- The inbound Axum API supports:
- - `POST /webhooks`
- - `PATCH /webhooks/{webhook_id}`
- - `POST /webhooks/{webhook_id}/validate`
-- Validation sends a signed `webhook.validation.test` event to the webhook endpoint and persists the result by updating `is_valid`.
-- Validation attempts are rate limited with the existing `rate_limit` crate. The validation key is `per-user-validate-webhook:{macro_user_id}:{webhook_id}` and the current limit is 10 attempts per 3600-second window.
+The inbound Axum API supports:
-## Typed filters
+- `POST /webhooks`
+- `PATCH /webhooks/{webhook_id}`
+- `DELETE /webhooks/{webhook_id}`
+- `POST /webhooks/{webhook_id}/validate`
-Webhooks use a typed `filters` array to decide which events are eligible for delivery. Each filter contains event names and may also restrict delivery to specific entity ids:
+Validation sends a signed `webhook.validation.test` event to the configured
+endpoint and persists the result in `is_valid`. Validation attempts use the
+existing `rate_limit` crate with the key
+`per-user-validate-webhook:{macro_user_id}:{webhook_id}` and a limit of 10
+attempts per 3600-second window.
+
+## Event ingestion and matching
+
+The `webhook-event-ingestion` Kafka consumer reads `macro.documents` and
+`macro.channels`. It supports these event names:
+
+- Documents: `document.created`, `document.updated`, `document.deleted`, and
+ `document.copied`.
+- Channels: `channel.created`, `channel.updated`, `channel.deleted`,
+ `channel.message_posted`, `channel.message_patched`,
+ `channel.message_deleted`, `channel.message_attachment_created`,
+ `channel.message_attachment_removed`, `channel.participant_added`, and
+ `channel.participant_removed`.
+
+For each event, ingestion asks `EntityAccessService` for the people who
+currently have access to its entity. The matching workspace set contains each
+person's Macro user ID, for personal webhooks, plus every team ID to which any
+of those people belongs. The set is deduplicated before matching, so one person
+or team is considered only once.
+
+A webhook matches only when it is active, valid, not soft-deleted, owned by one
+of those personal or team workspaces, and has one filter element that matches
+both the exact event name and entity ID. An event match in one filter element
+cannot be combined with an ID match in another element.
+
+### Filter-ID semantics
+
+Each filter requires `events`. `ids` is optional; an absent or `null` value
+matches every entity ID for that filter's events:
```json
[
{
"events": ["document.created"],
- "ids": ["doc_123"]
+ "ids": ["0197f776-6e7b-7c69-a251-780ae754d3e4"]
}
]
```
-`events` is required on every filter. `ids` is optional; when it is absent, that filter matches all entity ids for its events.
+For document events, IDs always mean the event's `document_id`. For every
+channel event, IDs mean `channel_id`. This includes message, attachment, and
+participant events: their message, attachment, and participant IDs are not used
+for webhook filtering.
+
+The matching query stores filters in the `webhook.filters` `JSONB NOT NULL`
+column, protected by the `webhook_filters_is_array` constraint. It uses the
+`webhook_filters_gin_idx` `jsonb_path_ops` GIN index for the event containment
+probe and an exact JSONPath recheck to enforce same-filter event/ID matching.
+
+Ingestion concurrently enqueues one message for every matched webhook and waits
+for every send. It succeeds only if all sends complete. Repository,
+workspace-resolution, queue, and database/internal access failures are
+transient: the consumer makes up to five in-process attempts with 1, 2, 4, and
+8-second delays. If all five fail, it exits without committing the Kafka offset;
+the service supervisor restarts it and Kafka redelivers the event. Invalid
+entity IDs, invalid broker contracts, empty Kafka payloads, and undecodable
+Kafka messages are traced and skipped so they do not block a partition.
+
+## Queue contract, deduplication, and ordering
+
+The versioned SQS message contains a webhook ID and a normalized event. The
+event retains the broker event ID, schema version, event name, entity type and
+ID, entity ordering key, ingestion timestamp, and complete broker envelope.
+The normalized `occurred_at` value, persisted as `event_occurred_at`, is the
+time of webhook ingestion rather than an event-producer timestamp. Endpoint
+URLs, custom headers, and signing secrets are deliberately omitted; the worker
+reloads the webhook's current configuration before delivery.
+
+The webhook event queue is FIFO. Each send explicitly uses:
-## Database storage and matching
+- `MessageGroupId`: the webhook ID.
+- `MessageDeduplicationId`: `<webhook_id>:<event_id>`.
-Filters are stored in the `webhook.filters` `JSONB NOT NULL` column. The column has the `webhook_filters_is_array` check constraint so stored filter data is always an array:
+This gives each webhook its own FIFO group and suppresses duplicate enqueue
+operations during SQS's deduplication interval. The database additionally has a
+unique `(webhook_id, event_id)` delivery constraint, so duplicate queue receipts
+reuse the same delivery record.
-```sql
-CHECK (jsonb_typeof(filters) = 'array')
+FIFO preserves the order in which the current consumer enqueues events for one
+webhook. It does not create a global order across Kafka partitions or across the
+`macro.documents` and `macro.channels` topics. Consumers must therefore treat
+the order as observed order, not total event order.
+
+## HTTP delivery contract
+
+Delivery is an HTTPS `POST` with a five-second timeout and redirects disabled.
+The complete broker envelope is serialized as the exact request body; normalized
+queue and delivery fields are not wrapped around or merged into it. Its shape is:
+
+```json
+{
+ "event_id": "0197f776-6e7b-7c69-a251-780ae754d3e4",
+ "schema_version": 1,
+ "event_type": "document.deleted",
+ "metadata": {
+ "document_id": "0197f776-6e7b-7c69-a251-780ae754d3e4",
+ "actor_user_id": null,
+ "project_id": null
+ }
+}
```
-The `webhook_filters_gin_idx` index is a GIN index using `jsonb_path_ops` on `filters`. It supports containment probes on keys inside filter elements, including `events` and `ids`. The canonical matching query used by `WebhookRepo::list_active_webhooks_matching_event` combines an index-backed containment predicate for the event with an exact JSONPath recheck for the same filter element's event and entity-id semantics:
+The request includes:
+
+- `Content-Type: application/json`
+- `X-Macro-Event`: the exact event name
+- `X-Macro-Event-Id`: the broker event ID
+- `X-Macro-Timestamp`: the request time as Unix seconds
+- `X-Macro-Signature`: `v1=<hex HMAC-SHA256>`
+
+The signature input is the raw byte sequence
+`<X-Macro-Timestamp>.<request-body>`, using the webhook signing secret. Custom
+headers are included, but case-insensitive attempts to override a Macro header
+are permanent configuration failures. Delivery applies the same HTTPS, DNS,
+blocked-address, reserved-header, signing, and no-redirect protections as
+validation.
+
+Every 2xx response succeeds. Network errors, timeouts, DNS resolution failures,
+and HTTP 408, 429, and 5xx responses are retryable. Other non-2xx responses and
+invalid endpoint or header configuration are permanent failures. Delivery
+attempts store response status, redacted response-header values, sanitized error
+details, duration, and at most 4096 bytes of a UTF-8-lossy response-body preview.
+
+## Delivery state, retries, and cancellation
+
+The worker prepares a delivery idempotently, reloads the webhook, and records
+each HTTP attempt in Postgres. Terminal delivered, canceled, permanently failed,
+and exhausted deliveries are acknowledged without another HTTP request.
+Successful attempts update `webhook.last_success_at`; attempted failures update
+`webhook.last_failure_at`.
+
+A delivery receives at most five HTTP attempts. Retryable failures use this
+schedule:
+
+| Failed attempt | Delay before next attempt |
+| --- | --- |
+| 1 | 30 seconds |
+| 2 | 60 seconds |
+| 3 | 120 seconds |
+| 4 | 300 seconds |
+| 5 | No retry; mark exhausted |
+
+A permanent failure is recorded and acknowledged immediately. If the webhook is
+missing, the message is acknowledged. If its current configuration is paused,
+disabled, invalid, or soft-deleted, the delivery is marked canceled where
+possible and acknowledged without consuming an HTTP attempt.
+
+Delivery failures update tracking data but do not automatically pause or
+disable a webhook. Automatic pause/disable policy is intentionally deferred;
+operators or webhook owners must currently change status explicitly.
+
+## Worker and at-least-once behavior
+
+The worker long-polls in batches and processes each received batch sequentially.
+A missing body, malformed JSON body, or unsupported queue contract version is a
+poison message: it is traced and deleted immediately when SQS supplied a receipt
+handle. A service, delete, or visibility-update failure is traced and leaves the
+message for SQS redelivery. In deployed environments, repeatedly unacknowledged
+messages move to the FIFO dead-letter queue after 20 receives.
+
+The pipeline is at-least-once. SQS deduplication and the database uniqueness
+constraint suppress ordinary duplicates, and a terminal database record
+prevents another request. There is still an unavoidable crash window: an
+endpoint can accept the HTTP request before the process records the result. A
+redelivery can then recover the interrupted attempt and call the endpoint again.
+Webhook receivers should use `X-Macro-Event-Id` as an idempotency key.
+
+## Operations
+
+### Queue configuration and local setup
+
+`macro_queues::WebhookEventQueue` selects these FIFO queue names:
+
+| Environment | Queue name |
+| --- | --- |
+| Local | `webhook-event-queue.fifo` |
+| Develop | `webhook-event-queue-dev.fifo` |
+| Production | `webhook-event-queue-prod.fifo` |
+
+`OVERRIDE_WEBHOOK_EVENT_QUEUE` takes precedence over the environment default.
+The webhook crate's `outbound::SqsWebhookQueue` wraps the shared
+`sqs_client::SQS` and owns webhook-specific FIFO serialization, polling,
+acknowledgment, and visibility changes.
+Set the override to the full queue URL when connecting to LocalStack. From the
+repository root, the canonical local orchestrator provisions the FIFO queue and
+exports this value automatically:
+
+```sh
+just run_local --no-frontend
+```
-```sql
-WHERE w.filters @> jsonb_build_array(jsonb_build_object('events', jsonb_build_array($2::text)))
- AND jsonb_path_exists(
- w.filters,
- '$[*] ? (@.events[*] == $event && (!exists (@.ids) || @.ids == null || @.ids[*] == $id))',
- jsonb_build_object('event', $2::text, 'id', $3::text)
- )
+Services inside the local compose network receive
+`http://localstack:4566/000000000000/webhook-event-queue.fifo`. The
+`docker-compose.local-e2e.yml` environment supplies the same override. Worker
+polling defaults to 10 messages and a 20-second long poll and can be adjusted
+with `WEBHOOK_QUEUE_MAX_MESSAGES` and `WEBHOOK_QUEUE_WAIT_TIME_SECONDS`.
+
+### Tracing
+
+Use a targeted `RUST_LOG` filter while diagnosing ingestion or delivery, for
+example:
+
+```text
+RUST_LOG=info,webhook=trace,webhook::outbound::sqs_queue=trace
```
-## Temporary limitations
+Ingestion spans expose event ID/name, entity type/ID, accessor count, workspace
+count, match count, and webhook ID. Worker and delivery spans expose queue
+message ID, webhook/event/delivery/attempt IDs, attempt number, delivery status,
+HTTP status, duration, and retry delay. Queue polling and acknowledgment errors
+are also traced. Request and response bodies, signing secrets, signatures, and
+custom header values must not be added to logs; the implemented spans skip
+those values.
-- Webhook IDs currently use prefixed UUIDv7 strings (`wh_<uuid_v7>`). They can be swapped for true ULIDs later.
-- Webhook signing secrets are generated by the service and stored in plaintext in `signing_secret`, matching the existing bot secret storage approach.
-- Custom headers are stored as JSON in the `headers` column.
+For durable investigation, correlate `webhook_delivery` and
+`webhook_delivery_attempt` rows by webhook/event ID and inspect the FIFO DLQ for
+transport failures that survived 20 receives.
-## Intentionally out of scope
+## Temporary limitations
-Delivery workers, SQS/SNS infrastructure, event ingestion, redelivery, and auto-pause behavior are intentionally out of scope for this crate iteration.
+- Webhook, delivery, and attempt IDs use prefixed UUIDv7 strings rather than true
+ ULIDs.
+- Webhook signing secrets are generated by the service and stored in plaintext
+ in `signing_secret`, matching the existing bot secret storage approach.
+- Custom headers are stored as JSON in the `headers` column; persisted delivery
+ snapshots and response headers redact their values.
diff --git a/rust/cloud-storage/webhook/src/domain/delivery.rs b/rust/cloud-storage/webhook/src/domain/delivery.rs
new file mode 100644
index 0000000000..ceba8f9265
--- /dev/null
+++ b/rust/cloud-storage/webhook/src/domain/delivery.rs
@@ -0,0 +1,288 @@
+//! Webhook event delivery state machine and retry policy.
+
+#[cfg(test)]
+mod test;
+
+use super::{
+ models::{
+ PreparedWebhookDelivery, WebhookDeliveryAttempt, WebhookDeliveryStatus,
+ WebhookEventQueueMessage, WebhookHttpOutcome, WebhookHttpOutcomeDetails, WebhookStatus,
+ WebhookWorkerDisposition,
+ },
+ ports::{WebhookDeliveryClient, WebhookDeliveryRepository, WebhookEventDeliveryService},
+};
+use chrono::Utc;
+use std::time::Duration;
+
+const MAX_HTTP_ATTEMPTS: u32 = 5;
+const RETRY_DELAYS: [Duration; 4] = [
+ Duration::from_secs(30),
+ Duration::from_secs(60),
+ Duration::from_secs(120),
+ Duration::from_secs(300),
+];
+
+/// Error returned while processing a queued webhook event.
+#[derive(Debug, thiserror::Error)]
+pub enum WebhookEventDeliveryError {
+ /// The delivery repository failed to prepare or persist delivery state.
+ #[error("webhook delivery repository failed: {0}")]
+ Repository(#[source] anyhow::Error),
+ /// The HTTP client encountered an internal failure with no delivery outcome.
+ #[error("webhook delivery client failed: {0}")]
+ Client(#[source] anyhow::Error),
+ /// A non-terminal delivery has already reached the domain's attempt limit.
+ #[error("webhook delivery has already started {0} attempts; maximum is five")]
+ AttemptLimitReached(u32),
+ /// The repository returned an attempt outside the domain's one-based limit.
+ #[error("webhook delivery repository returned invalid attempt number {0}")]
+ InvalidAttemptNumber(u32),
+ /// The repository declined a due, non-terminal delivery without exposing updated state.
+ #[error("webhook delivery repository did not begin a due delivery attempt")]
+ AttemptNotStarted,
+}
+
+/// Domain implementation of webhook event delivery and retry policy.
+#[derive(Debug, Clone)]
+pub struct WebhookEventDeliveryServiceImpl<R, C> {
+ repository: R,
+ client: C,
+}
+
+impl<R, C> WebhookEventDeliveryServiceImpl<R, C> {
+ /// Create a webhook event delivery service.
+ pub fn new(repository: R, client: C) -> Self {
+ Self { repository, client }
+ }
+}
+
+impl<R, C> WebhookEventDeliveryServiceImpl<R, C>
+where
+ R: WebhookDeliveryRepository,
+ C: WebhookDeliveryClient,
+{
+ fn repository_error(error: R::Err) -> WebhookEventDeliveryError {
+ WebhookEventDeliveryError::Repository(error.into())
+ }
+
+ fn client_error(error: C::Err) -> WebhookEventDeliveryError {
+ WebhookEventDeliveryError::Client(error.into())
+ }
+
+ async fn cancel_ineligible_delivery(
+ &self,
+ delivery_id: &str,
+ ) -> Result<WebhookWorkerDisposition, WebhookEventDeliveryError> {
+ self.repository
+ .cancel_delivery(delivery_id)
+ .await
+ .map_err(Self::repository_error)?;
+ record_delivery_status(WebhookDeliveryStatus::Canceled.as_str());
+ Ok(WebhookWorkerDisposition::Acknowledge)
+ }
+
+ async fn disposition_after_declined_attempt(
+ &self,
+ message: &WebhookEventQueueMessage,
+ ) -> Result<WebhookWorkerDisposition, WebhookEventDeliveryError> {
+ let Some(prepared) = self
+ .repository
+ .prepare_delivery(message)
+ .await
+ .map_err(Self::repository_error)?
+ else {
+ record_delivery_status("missing");
+ return Ok(WebhookWorkerDisposition::Acknowledge);
+ };
+
+ tracing::Span::current().record("delivery_id", prepared.delivery_id.as_str());
+ record_delivery_status(prepared.status.as_str());
+
+ if prepared.status.is_terminal() {
+ return Ok(WebhookWorkerDisposition::Acknowledge);
+ }
+ if !webhook_is_eligible(&prepared) {
+ return self.cancel_ineligible_delivery(&prepared.delivery_id).await;
+ }
+ if let Some(delay) = remaining_delay(&prepared) {
+ record_retry_delay(delay);
+ return Ok(WebhookWorkerDisposition::RetryAfter(delay));
+ }
+
+ Err(WebhookEventDeliveryError::AttemptNotStarted)
+ }
+
+ async fn record_outcome(
+ &self,
+ attempt: &WebhookDeliveryAttempt,
+ outcome: WebhookHttpOutcome,
+ ) -> Result<WebhookWorkerDisposition, WebhookEventDeliveryError> {
+ match outcome {
+ WebhookHttpOutcome::Success(details) => {
+ record_http_details(&details);
+ self.repository
+ .record_success(attempt, &details)
+ .await
+ .map_err(Self::repository_error)?;
+ record_delivery_status(WebhookDeliveryStatus::Delivered.as_str());
+ Ok(WebhookWorkerDisposition::Acknowledge)
+ }
+ WebhookHttpOutcome::PermanentFailure(details) => {
+ record_http_details(&details);
+ self.repository
+ .record_permanent_failure(attempt, &details)
+ .await
+ .map_err(Self::repository_error)?;
+ record_delivery_status(WebhookDeliveryStatus::PermanentlyFailed.as_str());
+ Ok(WebhookWorkerDisposition::Acknowledge)
+ }
+ WebhookHttpOutcome::RetryableFailure(details)
+ if attempt.attempt_number == MAX_HTTP_ATTEMPTS =>
+ {
+ record_http_details(&details);
+ self.repository
+ .record_exhaustion(attempt, &details)
+ .await
+ .map_err(Self::repository_error)?;
+ record_delivery_status(WebhookDeliveryStatus::Exhausted.as_str());
+ Ok(WebhookWorkerDisposition::Acknowledge)
+ }
+ WebhookHttpOutcome::RetryableFailure(details) => {
+ record_http_details(&details);
+ let delay = retry_delay(attempt.attempt_number).ok_or(
+ WebhookEventDeliveryError::InvalidAttemptNumber(attempt.attempt_number),
+ )?;
+ let next_attempt_at = Utc::now()
+ + chrono::Duration::from_std(delay)
+ .expect("webhook retry delays fit in chrono::Duration");
+
+ self.repository
+ .record_retryable_failure(attempt, &details, next_attempt_at)
+ .await
+ .map_err(Self::repository_error)?;
+ record_delivery_status(WebhookDeliveryStatus::RetryScheduled.as_str());
+ record_retry_delay(delay);
+ Ok(WebhookWorkerDisposition::RetryAfter(delay))
+ }
+ }
+ }
+}
+
+impl<R, C> WebhookEventDeliveryService for WebhookEventDeliveryServiceImpl<R, C>
+where
+ R: WebhookDeliveryRepository,
+ C: WebhookDeliveryClient,
+{
+ type Err = WebhookEventDeliveryError;
+
+ #[tracing::instrument(
+ skip(self, message),
+ fields(
+ webhook_id = %message.webhook_id,
+ event_id = %message.event.event_id,
+ delivery_id = tracing::field::Empty,
+ attempt_id = tracing::field::Empty,
+ attempt_number = tracing::field::Empty,
+ status = tracing::field::Empty,
+ http_status = tracing::field::Empty,
+ duration_ms = tracing::field::Empty,
+ retry_delay_seconds = tracing::field::Empty,
+ ),
+ err
+ )]
+ async fn deliver_event(
+ &self,
+ message: WebhookEventQueueMessage,
+ ) -> Result<WebhookWorkerDisposition, Self::Err> {
+ let Some(prepared) = self
+ .repository
+ .prepare_delivery(&message)
+ .await
+ .map_err(Self::repository_error)?
+ else {
+ record_delivery_status("missing");
+ return Ok(WebhookWorkerDisposition::Acknowledge);
+ };
+
+ tracing::Span::current().record("delivery_id", prepared.delivery_id.as_str());
+ record_delivery_status(prepared.status.as_str());
+
+ if prepared.status.is_terminal() {
+ return Ok(WebhookWorkerDisposition::Acknowledge);
+ }
+ if !webhook_is_eligible(&prepared) {
+ return self.cancel_ineligible_delivery(&prepared.delivery_id).await;
+ }
+ if let Some(delay) = remaining_delay(&prepared) {
+ record_retry_delay(delay);
+ return Ok(WebhookWorkerDisposition::RetryAfter(delay));
+ }
+ if prepared.attempt_count >= MAX_HTTP_ATTEMPTS {
+ return Err(WebhookEventDeliveryError::AttemptLimitReached(
+ prepared.attempt_count,
+ ));
+ }
+
+ let Some(attempt) = self
+ .repository
+ .begin_attempt(&prepared.delivery_id)
+ .await
+ .map_err(Self::repository_error)?
+ else {
+ return self.disposition_after_declined_attempt(&message).await;
+ };
+
+ if !(1..=MAX_HTTP_ATTEMPTS).contains(&attempt.attempt_number) {
+ return Err(WebhookEventDeliveryError::InvalidAttemptNumber(
+ attempt.attempt_number,
+ ));
+ }
+
+ tracing::Span::current().record("attempt_id", attempt.attempt_id.as_str());
+ tracing::Span::current().record("attempt_number", attempt.attempt_number);
+ record_delivery_status(WebhookDeliveryStatus::InProgress.as_str());
+
+ let outcome = self
+ .client
+ .deliver(&prepared.webhook, &message.event)
+ .await
+ .map_err(Self::client_error)?;
+
+ self.record_outcome(&attempt, outcome).await
+ }
+}
+
+fn webhook_is_eligible(prepared: &PreparedWebhookDelivery) -> bool {
+ prepared.webhook.status == WebhookStatus::Active
+ && prepared.webhook.is_valid
+ && prepared.webhook.deleted_at.is_none()
+}
+
+fn retry_delay(attempt_number: u32) -> Option<Duration> {
+ let delay_index = usize::try_from(attempt_number.checked_sub(1)?).ok()?;
+ RETRY_DELAYS.get(delay_index).copied()
+}
+
+fn remaining_delay(prepared: &PreparedWebhookDelivery) -> Option<Duration> {
+ let next_attempt_at = prepared.next_attempt_at?;
+ next_attempt_at
+ .signed_duration_since(Utc::now())
+ .to_std()
+ .ok()
+}
+
+fn record_delivery_status(status: &str) {
+ tracing::Span::current().record("status", status);
+}
+
+fn record_http_details(details: &WebhookHttpOutcomeDetails) {
+ if let Some(status) = details.response_status {
+ tracing::Span::current().record("http_status", status);
+ }
+ let duration_ms = u64::try_from(details.duration.as_millis()).unwrap_or(u64::MAX);
+ tracing::Span::current().record("duration_ms", duration_ms);
+}
+
+fn record_retry_delay(delay: Duration) {
+ tracing::Span::current().record("retry_delay_seconds", delay.as_secs());
+}
diff --git a/rust/cloud-storage/webhook/src/domain/delivery/test.rs b/rust/cloud-storage/webhook/src/domain/delivery/test.rs
new file mode 100644
index 0000000000..da3149b60f
--- /dev/null
+++ b/rust/cloud-storage/webhook/src/domain/delivery/test.rs
@@ -0,0 +1,616 @@
+use super::*;
+use crate::domain::{
+ models::{
+ NormalizedWebhookEvent, Webhook, WebhookDeliveryAttempt, WebhookDeliveryStatus,
+ WebhookEventQueueMessage, WebhookHeaders, WebhookHttpOutcome, WebhookHttpOutcomeDetails,
+ WebhookStatus, WebhookWorkerDisposition,
+ },
+ ports::{WebhookDeliveryClient, WebhookDeliveryRepository, WebhookEventDeliveryService},
+};
+use chrono::{DateTime, Utc};
+use serde_json::json;
+use std::{
+ collections::{BTreeMap, VecDeque},
+ sync::{Arc, Mutex, MutexGuard},
+ time::Duration,
+};
+
+const WEBHOOK_ID: &str = "wh_delivery_service";
+const DELIVERY_ID: &str = "whd_delivery_service";
+const EVENT_ID: &str = "evt_delivery_service";
+
+fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
+ mutex.lock().expect("test mutex is not poisoned")
+}
+
+fn timestamp() -> DateTime<Utc> {
+ DateTime::parse_from_rfc3339("2026-01-02T03:04:05Z")
+ .expect("valid timestamp")
+ .with_timezone(&Utc)
+}
+
+fn message() -> WebhookEventQueueMessage {
+ WebhookEventQueueMessage::new(
+ WEBHOOK_ID.to_string(),
+ NormalizedWebhookEvent {
+ event_id: EVENT_ID.to_string(),
+ schema_version: 1,
+ event_name: "document.created".to_string(),
+ entity_type: "document".to_string(),
+ entity_id: "doc_123".to_string(),
+ ordering_key: "doc_123".to_string(),
+ occurred_at: timestamp(),
+ broker_envelope: json!({"event_id": EVENT_ID, "metadata": {"id": "doc_123"}}),
+ },
+ )
+}
+
+fn webhook(endpoint_url: &str) -> Webhook {
+ Webhook {
+ id: WEBHOOK_ID.to_string(),
+ workspace_id: "macro|[email protected]".to_string(),
+ name: "Delivery test".to_string(),
+ endpoint_url: endpoint_url.to_string(),
+ signing_secret: "whsec_current".to_string(),
+ headers: BTreeMap::new(),
+ status: WebhookStatus::Active,
+ is_valid: true,
+ created_by_user_id: "macro|[email protected]".to_string(),
+ created_at: timestamp(),
+ updated_at: timestamp(),
+ deleted_at: None,
+ filters: Vec::new(),
+ }
+}
+
+fn prepared(status: WebhookDeliveryStatus, attempt_count: u32) -> PreparedWebhookDelivery {
+ PreparedWebhookDelivery {
+ delivery_id: DELIVERY_ID.to_string(),
+ webhook: webhook("https://current.example.com/webhook"),
+ status,
+ attempt_count,
+ next_attempt_at: None,
+ }
+}
+
+fn attempt(attempt_number: u32) -> WebhookDeliveryAttempt {
+ WebhookDeliveryAttempt {
+ attempt_id: format!("wha_{attempt_number}"),
+ delivery_id: DELIVERY_ID.to_string(),
+ attempt_number,
+ }
+}
+
+fn details(response_status: Option<u16>) -> WebhookHttpOutcomeDetails {
+ WebhookHttpOutcomeDetails {
+ duration: Duration::from_millis(125),
+ response_status,
+ response_headers_redacted: Some(WebhookHeaders::from([(
+ "content-type".to_string(),
+ "[REDACTED]".to_string(),
+ )])),
+ response_body_preview: None,
+ error_kind: None,
+ error_message: None,
+ }
+}
+
+#[derive(Default)]
+struct MockRepositoryState {
+ prepared: Option<PreparedWebhookDelivery>,
+ prepared_responses: VecDeque<Option<PreparedWebhookDelivery>>,
+ begin_responses: VecDeque<Option<WebhookDeliveryAttempt>>,
+ prepare_calls: Vec<WebhookEventQueueMessage>,
+ begin_calls: Vec<String>,
+ canceled_delivery_ids: Vec<String>,
+ successes: Vec<(WebhookDeliveryAttempt, WebhookHttpOutcomeDetails)>,
+ retries: Vec<(
+ WebhookDeliveryAttempt,
+ WebhookHttpOutcomeDetails,
+ DateTime<Utc>,
+ )>,
+ permanent_failures: Vec<(WebhookDeliveryAttempt, WebhookHttpOutcomeDetails)>,
+ exhaustions: Vec<(WebhookDeliveryAttempt, WebhookHttpOutcomeDetails)>,
+ fail_prepare: bool,
+ fail_begin: bool,
+ fail_record: bool,
+}
+
+#[derive(Clone, Default)]
+struct MockRepository {
+ state: Arc<Mutex<MockRepositoryState>>,
+}
+
+impl MockRepository {
+ fn for_attempt(prepared: PreparedWebhookDelivery, attempt_number: u32) -> Self {
+ let repository = Self::default();
+ {
+ let mut state = lock(&repository.state);
+ state.prepared = Some(prepared);
+ state
+ .begin_responses
+ .push_back(Some(attempt(attempt_number)));
+ }
+ repository
+ }
+}
+
+impl WebhookDeliveryRepository for MockRepository {
+ type Err = anyhow::Error;
+
+ async fn prepare_delivery(
+ &self,
+ message: &WebhookEventQueueMessage,
+ ) -> Result<Option<PreparedWebhookDelivery>, Self::Err> {
+ let mut state = lock(&self.state);
+ state.prepare_calls.push(message.clone());
+ if state.fail_prepare {
+ anyhow::bail!("prepare failed");
+ }
+ if let Some(response) = state.prepared_responses.pop_front() {
+ return Ok(response);
+ }
+ Ok(state.prepared.clone())
+ }
+
+ async fn cancel_delivery(&self, delivery_id: &str) -> Result<(), Self::Err> {
+ lock(&self.state)
+ .canceled_delivery_ids
+ .push(delivery_id.to_string());
+ Ok(())
+ }
+
+ async fn begin_attempt(
+ &self,
+ delivery_id: &str,
+ ) -> Result<Option<WebhookDeliveryAttempt>, Self::Err> {
+ let mut state = lock(&self.state);
+ state.begin_calls.push(delivery_id.to_string());
+ if state.fail_begin {
+ anyhow::bail!("begin failed");
+ }
+ Ok(state.begin_responses.pop_front().flatten())
+ }
+
+ async fn record_success(
+ &self,
+ attempt: &WebhookDeliveryAttempt,
+ details: &WebhookHttpOutcomeDetails,
+ ) -> Result<(), Self::Err> {
+ let mut state = lock(&self.state);
+ if state.fail_record {
+ anyhow::bail!("record failed");
+ }
+ state.successes.push((attempt.clone(), details.clone()));
+ Ok(())
+ }
+
+ async fn record_retryable_failure(
+ &self,
+ attempt: &WebhookDeliveryAttempt,
+ details: &WebhookHttpOutcomeDetails,
+ next_attempt_at: DateTime<Utc>,
+ ) -> Result<(), Self::Err> {
+ let mut state = lock(&self.state);
+ if state.fail_record {
+ anyhow::bail!("record failed");
+ }
+ state
+ .retries
+ .push((attempt.clone(), details.clone(), next_attempt_at));
+ Ok(())
+ }
+
+ async fn record_permanent_failure(
+ &self,
+ attempt: &WebhookDeliveryAttempt,
+ details: &WebhookHttpOutcomeDetails,
+ ) -> Result<(), Self::Err> {
+ let mut state = lock(&self.state);
+ if state.fail_record {
+ anyhow::bail!("record failed");
+ }
+ state
+ .permanent_failures
+ .push((attempt.clone(), details.clone()));
+ Ok(())
+ }
+
+ async fn record_exhaustion(
+ &self,
+ attempt: &WebhookDeliveryAttempt,
+ details: &WebhookHttpOutcomeDetails,
+ ) -> Result<(), Self::Err> {
+ let mut state = lock(&self.state);
+ if state.fail_record {
+ anyhow::bail!("record failed");
+ }
+ state.exhaustions.push((attempt.clone(), details.clone()));
+ Ok(())
+ }
+}
+
+enum MockClientResponse {
+ Outcome(WebhookHttpOutcome),
+ Error,
+}
+
+#[derive(Default)]
+struct MockClientState {
+ responses: VecDeque<MockClientResponse>,
+ requests: Vec<(Webhook, NormalizedWebhookEvent)>,
+}
+
+#[derive(Clone, Default)]
+struct MockClient {
+ state: Arc<Mutex<MockClientState>>,
+}
+
+impl MockClient {
+ fn with_outcome(outcome: WebhookHttpOutcome) -> Self {
+ let client = Self::default();
+ lock(&client.state)
+ .responses
+ .push_back(MockClientResponse::Outcome(outcome));
+ client
+ }
+}
+
+impl WebhookDeliveryClient for MockClient {
+ type Err = anyhow::Error;
+
+ async fn deliver(
+ &self,
+ webhook: &Webhook,
+ event: &NormalizedWebhookEvent,
+ ) -> Result<WebhookHttpOutcome, Self::Err> {
+ let mut state = lock(&self.state);
+ state.requests.push((webhook.clone(), event.clone()));
+ match state.responses.pop_front() {
+ Some(MockClientResponse::Outcome(outcome)) => Ok(outcome),
+ Some(MockClientResponse::Error) => anyhow::bail!("internal client failure"),
+ None => anyhow::bail!("no mock client response"),
+ }
+ }
+}
+
+#[tokio::test]
+async fn successful_delivery_is_recorded_and_acknowledged() {
+ let repository = MockRepository::for_attempt(prepared(WebhookDeliveryStatus::Queued, 0), 1);
+ let success_details = details(Some(204));
+ let client = MockClient::with_outcome(WebhookHttpOutcome::Success(success_details.clone()));
+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client.clone());
+
+ let disposition = service.deliver_event(message()).await.expect("delivery");
+
+ assert_eq!(disposition, WebhookWorkerDisposition::Acknowledge);
+ let repository_state = lock(&repository.state);
+ assert_eq!(
+ repository_state.successes,
+ vec![(attempt(1), success_details)]
+ );
+ assert!(repository_state.retries.is_empty());
+ assert_eq!(lock(&client.state).requests.len(), 1);
+}
+
+#[tokio::test]
+async fn permanent_failure_is_recorded_and_acknowledged() {
+ let repository = MockRepository::for_attempt(prepared(WebhookDeliveryStatus::Queued, 0), 1);
+ let failure_details = details(Some(400));
+ let client = MockClient::with_outcome(WebhookHttpOutcome::PermanentFailure(
+ failure_details.clone(),
+ ));
+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client);
+
+ let disposition = service.deliver_event(message()).await.expect("delivery");
+
+ assert_eq!(disposition, WebhookWorkerDisposition::Acknowledge);
+ assert_eq!(
+ lock(&repository.state).permanent_failures,
+ vec![(attempt(1), failure_details)]
+ );
+}
+
+#[tokio::test]
+async fn retryable_failures_use_the_exact_delay_schedule() {
+ for (attempt_number, expected_seconds) in [(1, 30), (2, 60), (3, 120), (4, 300)] {
+ let repository = MockRepository::for_attempt(
+ prepared(
+ if attempt_number == 1 {
+ WebhookDeliveryStatus::Queued
+ } else {
+ WebhookDeliveryStatus::RetryScheduled
+ },
+ attempt_number - 1,
+ ),
+ attempt_number,
+ );
+ let failure_details = details(Some(503));
+ let client = MockClient::with_outcome(WebhookHttpOutcome::RetryableFailure(
+ failure_details.clone(),
+ ));
+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client);
+ let expected_delay = Duration::from_secs(expected_seconds);
+ let earliest_due =
+ Utc::now() + chrono::Duration::from_std(expected_delay).expect("valid test delay");
+
+ let disposition = service.deliver_event(message()).await.expect("delivery");
+
+ let latest_due =
+ Utc::now() + chrono::Duration::from_std(expected_delay).expect("valid test delay");
+ assert_eq!(
+ disposition,
+ WebhookWorkerDisposition::RetryAfter(expected_delay)
+ );
+ let repository_state = lock(&repository.state);
+ assert_eq!(repository_state.retries.len(), 1);
+ let (recorded_attempt, recorded_details, next_attempt_at) = &repository_state.retries[0];
+ assert_eq!(recorded_attempt, &attempt(attempt_number));
+ assert_eq!(recorded_details, &failure_details);
+ assert!(*next_attempt_at >= earliest_due);
+ assert!(*next_attempt_at <= latest_due);
+ assert!(repository_state.exhaustions.is_empty());
+ }
+}
+
+#[tokio::test]
+async fn fifth_retryable_failure_is_exhausted_and_acknowledged() {
+ let repository =
+ MockRepository::for_attempt(prepared(WebhookDeliveryStatus::RetryScheduled, 4), 5);
+ let failure_details = details(Some(429));
+ let client = MockClient::with_outcome(WebhookHttpOutcome::RetryableFailure(
+ failure_details.clone(),
+ ));
+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client);
+
+ let disposition = service.deliver_event(message()).await.expect("delivery");
+
+ assert_eq!(disposition, WebhookWorkerDisposition::Acknowledge);
+ let repository_state = lock(&repository.state);
+ assert_eq!(
+ repository_state.exhaustions,
+ vec![(attempt(5), failure_details)]
+ );
+ assert!(repository_state.retries.is_empty());
+}
+
+#[tokio::test]
+async fn terminal_deliveries_suppress_duplicate_http_attempts() {
+ for status in [
+ WebhookDeliveryStatus::Delivered,
+ WebhookDeliveryStatus::Canceled,
+ WebhookDeliveryStatus::PermanentlyFailed,
+ WebhookDeliveryStatus::Exhausted,
+ ] {
+ let repository = MockRepository::default();
+ lock(&repository.state).prepared = Some(prepared(status, 1));
+ let client = MockClient::default();
+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client.clone());
+
+ let disposition = service.deliver_event(message()).await.expect("delivery");
+
+ assert_eq!(disposition, WebhookWorkerDisposition::Acknowledge);
+ assert!(lock(&repository.state).begin_calls.is_empty());
+ assert!(lock(&client.state).requests.is_empty());
+ }
+}
+
+#[tokio::test]
+async fn every_due_non_terminal_status_can_start_an_attempt() {
+ for (status, attempt_count, attempt_number) in [
+ (WebhookDeliveryStatus::Queued, 0, 1),
+ (WebhookDeliveryStatus::InProgress, 1, 2),
+ (WebhookDeliveryStatus::RetryScheduled, 2, 3),
+ ] {
+ let repository =
+ MockRepository::for_attempt(prepared(status, attempt_count), attempt_number);
+ let client = MockClient::with_outcome(WebhookHttpOutcome::Success(details(Some(200))));
+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client);
+
+ let disposition = service.deliver_event(message()).await.expect("delivery");
+
+ assert_eq!(disposition, WebhookWorkerDisposition::Acknowledge);
+ assert_eq!(lock(&repository.state).successes.len(), 1);
+ }
+}
+
+#[tokio::test]
+async fn future_retry_returns_its_remaining_delay_without_an_attempt() {
+ let repository = MockRepository::default();
+ let due_at = Utc::now() + chrono::Duration::hours(1);
+ let mut future_delivery = prepared(WebhookDeliveryStatus::RetryScheduled, 1);
+ future_delivery.next_attempt_at = Some(due_at);
+ lock(&repository.state).prepared = Some(future_delivery);
+ let client = MockClient::default();
+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client.clone());
+
+ let disposition = service.deliver_event(message()).await.expect("delivery");
+
+ let WebhookWorkerDisposition::RetryAfter(delay) = disposition else {
+ panic!("future delivery should be retried later");
+ };
+ assert!(delay <= Duration::from_secs(3600));
+ assert!(delay > Duration::from_secs(3599));
+ assert!(lock(&repository.state).begin_calls.is_empty());
+ assert!(lock(&client.state).requests.is_empty());
+}
+
+#[tokio::test]
+async fn ineligible_webhooks_are_canceled_without_http_attempts() {
+ let mut ineligible_webhooks = Vec::new();
+
+ let mut paused = webhook("https://paused.example.com");
+ paused.status = WebhookStatus::Paused;
+ ineligible_webhooks.push(paused);
+
+ let mut disabled = webhook("https://disabled.example.com");
+ disabled.status = WebhookStatus::Disabled;
+ ineligible_webhooks.push(disabled);
+
+ let mut invalid = webhook("https://invalid.example.com");
+ invalid.is_valid = false;
+ ineligible_webhooks.push(invalid);
+
+ let mut deleted = webhook("https://deleted.example.com");
+ deleted.deleted_at = Some(Utc::now());
+ ineligible_webhooks.push(deleted);
+
+ for ineligible_webhook in ineligible_webhooks {
+ let repository = MockRepository::default();
+ let mut delivery = prepared(WebhookDeliveryStatus::Queued, 0);
+ delivery.webhook = ineligible_webhook;
+ lock(&repository.state).prepared = Some(delivery);
+ let client = MockClient::default();
+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client.clone());
+
+ let disposition = service.deliver_event(message()).await.expect("delivery");
+
+ assert_eq!(disposition, WebhookWorkerDisposition::Acknowledge);
+ let repository_state = lock(&repository.state);
+ assert_eq!(
+ repository_state.canceled_delivery_ids,
+ vec![DELIVERY_ID.to_string()]
+ );
+ assert!(repository_state.begin_calls.is_empty());
+ assert!(lock(&client.state).requests.is_empty());
+ }
+}
+
+#[tokio::test]
+async fn missing_webhook_is_acknowledged_without_an_http_attempt() {
+ let repository = MockRepository::default();
+ let client = MockClient::default();
+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client.clone());
+
+ let disposition = service.deliver_event(message()).await.expect("delivery");
+
+ assert_eq!(disposition, WebhookWorkerDisposition::Acknowledge);
+ let repository_state = lock(&repository.state);
+ assert!(repository_state.canceled_delivery_ids.is_empty());
+ assert!(repository_state.begin_calls.is_empty());
+ assert!(lock(&client.state).requests.is_empty());
+}
+
+#[tokio::test]
+async fn each_queue_receipt_uses_refreshed_webhook_configuration() {
+ let repository = MockRepository::default();
+ let mut first = prepared(WebhookDeliveryStatus::Queued, 0);
+ first.webhook.endpoint_url = "https://old.example.com/webhook".to_string();
+ first.webhook.signing_secret = "whsec_old".to_string();
+ let mut refreshed = prepared(WebhookDeliveryStatus::Queued, 0);
+ refreshed.webhook.endpoint_url = "https://new.example.com/webhook".to_string();
+ refreshed.webhook.signing_secret = "whsec_new".to_string();
+ refreshed.webhook.headers = BTreeMap::from([("X-Current".to_string(), "yes".to_string())]);
+ {
+ let mut state = lock(&repository.state);
+ state.prepared_responses.push_back(Some(first));
+ state.prepared_responses.push_back(Some(refreshed));
+ state.begin_responses.push_back(Some(attempt(1)));
+ state.begin_responses.push_back(Some(attempt(1)));
+ }
+
+ let client = MockClient::default();
+ {
+ let mut state = lock(&client.state);
+ state
+ .responses
+ .push_back(MockClientResponse::Outcome(WebhookHttpOutcome::Success(
+ details(Some(200)),
+ )));
+ state
+ .responses
+ .push_back(MockClientResponse::Outcome(WebhookHttpOutcome::Success(
+ details(Some(200)),
+ )));
+ }
+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client.clone());
+
+ service
+ .deliver_event(message())
+ .await
+ .expect("first receipt");
+ service
+ .deliver_event(message())
+ .await
+ .expect("second receipt");
+
+ assert_eq!(lock(&repository.state).prepare_calls.len(), 2);
+ let client_state = lock(&client.state);
+ assert_eq!(client_state.requests.len(), 2);
+ assert_eq!(
+ client_state.requests[0].0.endpoint_url,
+ "https://old.example.com/webhook"
+ );
+ assert_eq!(client_state.requests[0].0.signing_secret, "whsec_old");
+ assert_eq!(
+ client_state.requests[1].0.endpoint_url,
+ "https://new.example.com/webhook"
+ );
+ assert_eq!(client_state.requests[1].0.signing_secret, "whsec_new");
+ assert_eq!(
+ client_state.requests[1].0.headers.get("X-Current"),
+ Some(&"yes".to_string())
+ );
+}
+
+#[tokio::test]
+async fn repository_and_internal_client_failures_remain_unacknowledged() {
+ let prepare_failure_repository = MockRepository::default();
+ lock(&prepare_failure_repository.state).fail_prepare = true;
+ let prepare_failure_service =
+ WebhookEventDeliveryServiceImpl::new(prepare_failure_repository, MockClient::default());
+ assert!(matches!(
+ prepare_failure_service.deliver_event(message()).await,
+ Err(WebhookEventDeliveryError::Repository(_))
+ ));
+
+ let client_failure_repository =
+ MockRepository::for_attempt(prepared(WebhookDeliveryStatus::Queued, 0), 1);
+ let client = MockClient::default();
+ lock(&client.state)
+ .responses
+ .push_back(MockClientResponse::Error);
+ let client_failure_service =
+ WebhookEventDeliveryServiceImpl::new(client_failure_repository.clone(), client);
+ assert!(matches!(
+ client_failure_service.deliver_event(message()).await,
+ Err(WebhookEventDeliveryError::Client(_))
+ ));
+ let repository_state = lock(&client_failure_repository.state);
+ assert!(repository_state.successes.is_empty());
+ assert!(repository_state.retries.is_empty());
+ assert!(repository_state.permanent_failures.is_empty());
+ assert!(repository_state.exhaustions.is_empty());
+}
+
+#[tokio::test]
+async fn non_terminal_delivery_at_the_attempt_limit_does_not_start_a_sixth_attempt() {
+ let repository = MockRepository::for_attempt(
+ prepared(WebhookDeliveryStatus::InProgress, MAX_HTTP_ATTEMPTS),
+ MAX_HTTP_ATTEMPTS + 1,
+ );
+ let client = MockClient::with_outcome(WebhookHttpOutcome::Success(details(Some(200))));
+ let service = WebhookEventDeliveryServiceImpl::new(repository.clone(), client.clone());
+
+ assert!(matches!(
+ service.deliver_event(message()).await,
+ Err(WebhookEventDeliveryError::AttemptLimitReached(
+ MAX_HTTP_ATTEMPTS
+ ))
+ ));
+ assert!(lock(&repository.state).begin_calls.is_empty());
+ assert!(lock(&client.state).requests.is_empty());
+}
+
+#[tokio::test]
+async fn retry_is_returned_only_after_the_retry_state_is_recorded() {
+ let repository = MockRepository::for_attempt(prepared(WebhookDeliveryStatus::Queued, 0), 1);
+ lock(&repository.state).fail_record = true;
+ let client = MockClient::with_outcome(WebhookHttpOutcome::RetryableFailure(details(Some(503))));
+ let service = WebhookEventDeliveryServiceImpl::new(repository, client);
+
+ assert!(matches!(
+ service.deliver_event(message()).await,
+ Err(WebhookEventDeliveryError::Repository(_))
+ ));
+}
diff --git a/rust/cloud-storage/webhook/src/domain/ingestion.rs b/rust/cloud-storage/webhook/src/domain/ingestion.rs
index 3d6834a0e2..b25da39a5d 100644
--- a/rust/cloud-storage/webhook/src/domain/ingestion.rs
+++ b/rust/cloud-storage/webhook/src/domain/ingestion.rs
@@ -1,72 +1,83 @@
-//! Webhook event ingestion: broker events in, matching webhooks out.
+//! Webhook event ingestion: broker events in, matching queue messages out.
//!
//! `WebhookEventIngestionService` is the inbound port driven by broker
//! consumers (see `crate::inbound::kafka_consumer`). It is intentionally
//! separate from `WebhookService`, which is the CRUD surface for webhook
//! creation and edits.
-//!
-//! Event handling is not implemented yet: every handler below is a stub that
-//! only logs the event. The planned flow for each event is:
-//! 1. resolve the users that can see the event's entity via
-//! `EntityAccessService::get_users_by_entity` (see
-//! `WebhookEventIngestionServiceImpl::users_with_access`);
-//! 2. call `WebhookRepo::list_active_webhooks_matching_event` to load those
-//! users' active webhooks whose typed `filters` match the event name and
-//! entity id;
-//! 3. enqueue a delivery for every match.
#[cfg(test)]
mod test;
-use channels::domain::broker_events::{
- ChannelCreatedMetadata, ChannelDeletedMetadata, ChannelMessageAttachmentCreatedMetadata,
- ChannelMessageAttachmentRemovedMetadata, ChannelMessageDeletedMetadata,
- ChannelMessagePatchedMetadata, ChannelMessagePostedMetadata, ChannelParticipantAddedMetadata,
- ChannelParticipantRemovedMetadata, ChannelTopicEvent, ChannelUpdatedMetadata,
-};
-use documents::domain::events::{
- DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata, DocumentTopicEvent,
- DocumentUpdatedMetadata,
+use crate::domain::{
+ models::{NormalizedWebhookEvent, WebhookEventQueueMessage},
+ ports::{WebhookEventEnqueuer, WebhookRepo, WebhookWorkspaceResolver},
};
+use channels::domain::broker_events::ChannelTopicEvent;
+use chrono::Utc;
+use documents::domain::events::DocumentTopicEvent;
use entity_access::domain::models::{AccessError, EntityType};
use entity_access::domain::ports::EntityAccessService;
+use futures::future::join_all;
use macro_event_broker::Event;
use macro_user_id::user_id::MacroUserIdStr;
use std::future::Future;
use std::sync::Arc;
+use tracing::Instrument as _;
use uuid::Uuid;
+const DOCUMENT_ENTITY_TYPE: &str = "document";
+const CHANNEL_ENTITY_TYPE: &str = "channel";
+
/// Webhook event ingestion error.
#[derive(Debug, thiserror::Error)]
pub enum WebhookEventIngestionError {
/// Failed to resolve the users with access to the event's entity.
#[error(transparent)]
EntityAccess(#[from] AccessError),
- /// Repository or adapter error.
- #[error(transparent)]
- Internal(#[from] anyhow::Error),
+ /// The event's entity identifier violates the broker contract.
+ #[error("invalid {entity_type} entity id: {entity_id}")]
+ InvalidEntityId {
+ /// Contract entity type.
+ entity_type: &'static str,
+ /// Invalid entity identifier.
+ entity_id: String,
+ },
+ /// The broker envelope could not be represented by the queue contract.
+ #[error("failed to serialize broker envelope: {0}")]
+ Serialization(#[from] serde_json::Error),
+ /// Failed to resolve personal and team webhook workspaces.
+ #[error("failed to resolve webhook workspaces: {0}")]
+ WorkspaceResolution(#[source] anyhow::Error),
+ /// Failed to find active webhooks matching the event.
+ #[error("failed to match active webhooks: {0}")]
+ Repository(#[source] anyhow::Error),
+ /// Failed to enqueue at least one matched webhook event.
+ #[error("failed to enqueue webhook event: {0}")]
+ Enqueue(#[source] anyhow::Error),
}
impl WebhookEventIngestionError {
/// Whether retrying the same event could plausibly succeed.
///
- /// Database errors from access resolution are transient: the lookup can be
- /// retried once the database recovers, and the consumer holds off on
- /// committing the event's offset until it does. Everything else — invalid
- /// entity ids, unauthorized lookups, adapter bugs — is permanent, and
- /// retrying would only repeat the failure.
+ /// Access resolution currently maps some PostgreSQL failures to
+ /// [`AccessError::Internal`], so that variant remains retryable alongside
+ /// explicit database, repository, workspace-resolution, and queue errors.
+ /// Invalid broker contracts are permanent and can be safely skipped.
pub fn is_transient(&self) -> bool {
- match self {
- Self::EntityAccess(AccessError::DatabaseError(_)) => true,
- Self::EntityAccess(_) | Self::Internal(_) => false,
- }
+ matches!(
+ self,
+ Self::EntityAccess(AccessError::DatabaseError(_) | AccessError::Internal)
+ | Self::WorkspaceResolution(_)
+ | Self::Repository(_)
+ | Self::Enqueue(_)
+ )
}
}
/// Inbound port for ingesting broker events for webhook delivery.
///
/// One method per subscribed topic; each takes the decoded event envelope so
-/// implementations keep the `event_id` for delivery idempotency.
+/// implementations retain the `event_id` and complete original payload.
pub trait WebhookEventIngestionService: Clone + Send + Sync + 'static {
/// Ingest one `macro.documents` event envelope.
fn ingest_document_event(
@@ -83,254 +94,209 @@ pub trait WebhookEventIngestionService: Clone + Send + Sync + 'static {
/// Webhook event ingestion service implementation.
#[derive(Clone)]
-pub struct WebhookEventIngestionServiceImpl<A> {
+pub struct WebhookEventIngestionServiceImpl<A, R, Q> {
entity_access_service: Arc<A>,
+ repository: R,
+ enqueuer: Q,
}
-impl<A> WebhookEventIngestionServiceImpl<A> {
+impl<A, R, Q> WebhookEventIngestionServiceImpl<A, R, Q> {
/// Create a webhook event ingestion service.
- pub fn new(entity_access_service: Arc<A>) -> Self {
+ pub fn new(entity_access_service: Arc<A>, repository: R, enqueuer: Q) -> Self {
Self {
entity_access_service,
+ repository,
+ enqueuer,
}
}
}
-impl<A: EntityAccessService> WebhookEventIngestionServiceImpl<A> {
+impl<A, R, Q> WebhookEventIngestionServiceImpl<A, R, Q>
+where
+ A: EntityAccessService,
+ R: WebhookRepo + WebhookWorkspaceResolver,
+ Q: WebhookEventEnqueuer,
+{
/// Resolve the users that currently have access to an entity.
- ///
- /// This is the first step of webhook fan-out: the handlers below will use
- /// it to find the users whose webhooks should be matched against an event.
#[tracing::instrument(skip(self), err)]
pub async fn users_with_access(
&self,
entity_id: &str,
entity_type: EntityType,
) -> Result<Vec<MacroUserIdStr<'static>>, WebhookEventIngestionError> {
- tracing::trace!(entity_id, ?entity_type, "resolving users with access");
- let users = self
- .entity_access_service
+ self.entity_access_service
.get_users_by_entity(entity_id, entity_type)
- .await?;
- tracing::trace!(
- entity_id,
- user_count = users.len(),
- "resolved users with access"
- );
- Ok(users)
- }
-
- async fn handle_document_created(
- &self,
- event_id: Uuid,
- metadata: DocumentCreatedMetadata,
- ) -> Result<(), WebhookEventIngestionError> {
- tracing::debug!(
- %event_id,
- document_id = %metadata.document_id,
- "document.created: webhook ingestion not implemented yet"
- );
- Ok(())
+ .await
+ .map_err(WebhookEventIngestionError::EntityAccess)
}
- async fn handle_document_updated(
+ #[tracing::instrument(
+ skip_all,
+ fields(
+ event_id = %event.event_id,
+ event_name = %event.event_name,
+ entity_type = %event.entity_type,
+ entity_id = %event.entity_id,
+ accessor_count = tracing::field::Empty,
+ workspace_count = tracing::field::Empty,
+ match_count = tracing::field::Empty,
+ ),
+ err
+ )]
+ async fn ingest_normalized_event(
&self,
- event_id: Uuid,
- metadata: DocumentUpdatedMetadata,
- ) -> Result<(), WebhookEventIngestionError> {
- tracing::debug!(
- %event_id,
- document_id = %metadata.document_id,
- "document.updated: webhook ingestion not implemented yet"
- );
- Ok(())
- }
-
- async fn handle_document_deleted(
- &self,
- event_id: Uuid,
- metadata: DocumentDeletedMetadata,
+ event: NormalizedWebhookEvent,
+ entity_type: EntityType,
) -> Result<(), WebhookEventIngestionError> {
- tracing::debug!(
- %event_id,
- document_id = %metadata.document_id,
- "document.deleted: webhook ingestion not implemented yet"
- );
- Ok(())
- }
+ let accessors = self
+ .users_with_access(&event.entity_id, entity_type)
+ .await?;
+ tracing::Span::current().record("accessor_count", accessors.len());
- async fn handle_document_copied(
- &self,
- event_id: Uuid,
- metadata: DocumentCopiedMetadata,
- ) -> Result<(), WebhookEventIngestionError> {
- tracing::debug!(
- %event_id,
- document_id = %metadata.document_id,
- source_document_id = %metadata.source_document_id,
- "document.copied: webhook ingestion not implemented yet"
- );
- Ok(())
- }
+ let workspace_ids = self
+ .repository
+ .resolve_workspace_ids(accessors)
+ .await
+ .map_err(|error| WebhookEventIngestionError::WorkspaceResolution(error.into()))?;
+ tracing::Span::current().record("workspace_count", workspace_ids.len());
- async fn handle_channel_created(
- &self,
- event_id: Uuid,
- metadata: ChannelCreatedMetadata,
- ) -> Result<(), WebhookEventIngestionError> {
- tracing::debug!(
- %event_id,
- channel_id = %metadata.channel_id,
- "channel.created: webhook ingestion not implemented yet"
- );
- Ok(())
- }
+ let webhooks = self
+ .repository
+ .list_active_webhooks_matching_event(
+ workspace_ids,
+ event.event_name.clone(),
+ event.entity_id.clone(),
+ )
+ .await
+ .map_err(|error| WebhookEventIngestionError::Repository(error.into()))?;
+ tracing::Span::current().record("match_count", webhooks.len());
- async fn handle_channel_updated(
- &self,
- event_id: Uuid,
- metadata: ChannelUpdatedMetadata,
- ) -> Result<(), WebhookEventIngestionError> {
- tracing::debug!(
- %event_id,
- channel_id = %metadata.channel_id,
- "channel.updated: webhook ingestion not implemented yet"
- );
- Ok(())
- }
+ let enqueue_results = join_all(webhooks.into_iter().map(|webhook| {
+ let enqueuer = self.enqueuer.clone();
+ let webhook_id = webhook.id;
+ let message = WebhookEventQueueMessage::new(webhook_id.clone(), event.clone());
+ async move {
+ enqueuer
+ .enqueue(message)
+ .instrument(tracing::info_span!("enqueue_webhook_event", %webhook_id))
+ .await
+ .map_err(Into::into)
+ }
+ }))
+ .await;
- async fn handle_channel_deleted(
- &self,
- event_id: Uuid,
- metadata: ChannelDeletedMetadata,
- ) -> Result<(), WebhookEventIngestionError> {
- tracing::debug!(
- %event_id,
- channel_id = %metadata.channel_id,
- "channel.deleted: webhook ingestion not implemented yet"
- );
- Ok(())
- }
+ for result in enqueue_results {
+ result.map_err(WebhookEventIngestionError::Enqueue)?;
+ }
- async fn handle_channel_message_posted(
- &self,
- event_id: Uuid,
- metadata: ChannelMessagePostedMetadata,
- ) -> Result<(), WebhookEventIngestionError> {
- tracing::debug!(
- %event_id,
- channel_id = %metadata.channel_id,
- message_id = %metadata.message_id,
- "channel.message_posted: webhook ingestion not implemented yet"
- );
Ok(())
}
+}
- async fn handle_channel_message_patched(
- &self,
- event_id: Uuid,
- metadata: ChannelMessagePatchedMetadata,
- ) -> Result<(), WebhookEventIngestionError> {
- tracing::debug!(
- %event_id,
- channel_id = %metadata.channel_id,
- message_id = %metadata.message_id,
- "channel.message_patched: webhook ingestion not implemented yet"
- );
- Ok(())
- }
+fn normalized_document_event(
+ event: &Event<DocumentTopicEvent>,
+) -> Result<NormalizedWebhookEvent, WebhookEventIngestionError> {
+ let (event_name, entity_id) = match &event.event {
+ DocumentTopicEvent::Created(metadata) => ("document.created", &metadata.document_id),
+ DocumentTopicEvent::Updated(metadata) => ("document.updated", &metadata.document_id),
+ DocumentTopicEvent::Deleted(metadata) => ("document.deleted", &metadata.document_id),
+ DocumentTopicEvent::Copied(metadata) => ("document.copied", &metadata.document_id),
+ };
- async fn handle_channel_message_deleted(
- &self,
- event_id: Uuid,
- metadata: ChannelMessageDeletedMetadata,
- ) -> Result<(), WebhookEventIngestionError> {
- tracing::debug!(
- %event_id,
- channel_id = %metadata.channel_id,
- message_id = %metadata.message_id,
- "channel.message_deleted: webhook ingestion not implemented yet"
- );
- Ok(())
+ if Uuid::parse_str(entity_id).is_err() {
+ return Err(WebhookEventIngestionError::InvalidEntityId {
+ entity_type: DOCUMENT_ENTITY_TYPE,
+ entity_id: entity_id.clone(),
+ });
}
- async fn handle_channel_message_attachment_created(
- &self,
- event_id: Uuid,
- metadata: ChannelMessageAttachmentCreatedMetadata,
- ) -> Result<(), WebhookEventIngestionError> {
- tracing::debug!(
- %event_id,
- channel_id = %metadata.channel_id,
- message_id = %metadata.message_id,
- "channel.message_attachment_created: webhook ingestion not implemented yet"
- );
- Ok(())
- }
+ let broker_envelope = serde_json::to_value(event)?;
+ Ok(normalized_event(
+ event.event_id,
+ event.schema_version,
+ event_name,
+ DOCUMENT_ENTITY_TYPE,
+ entity_id,
+ broker_envelope,
+ ))
+}
- async fn handle_channel_message_attachment_removed(
- &self,
- event_id: Uuid,
- metadata: ChannelMessageAttachmentRemovedMetadata,
- ) -> Result<(), WebhookEventIngestionError> {
- tracing::debug!(
- %event_id,
- channel_id = %metadata.channel_id,
- message_id = %metadata.message_id,
- "channel.message_attachment_removed: webhook ingestion not implemented yet"
- );
- Ok(())
- }
+fn normalized_channel_event(
+ event: &Event<ChannelTopicEvent>,
+) -> Result<NormalizedWebhookEvent, WebhookEventIngestionError> {
+ let (event_name, channel_id) = match &event.event {
+ ChannelTopicEvent::Created(metadata) => ("channel.created", metadata.channel_id),
+ ChannelTopicEvent::Updated(metadata) => ("channel.updated", metadata.channel_id),
+ ChannelTopicEvent::Deleted(metadata) => ("channel.deleted", metadata.channel_id),
+ ChannelTopicEvent::MessagePosted(metadata) => {
+ ("channel.message_posted", metadata.channel_id)
+ }
+ ChannelTopicEvent::MessagePatched(metadata) => {
+ ("channel.message_patched", metadata.channel_id)
+ }
+ ChannelTopicEvent::MessageDeleted(metadata) => {
+ ("channel.message_deleted", metadata.channel_id)
+ }
+ ChannelTopicEvent::MessageAttachmentCreated(metadata) => {
+ ("channel.message_attachment_created", metadata.channel_id)
+ }
+ ChannelTopicEvent::MessageAttachmentRemoved(metadata) => {
+ ("channel.message_attachment_removed", metadata.channel_id)
+ }
+ ChannelTopicEvent::ParticipantAdded(metadata) => {
+ ("channel.participant_added", metadata.channel_id)
+ }
+ ChannelTopicEvent::ParticipantRemoved(metadata) => {
+ ("channel.participant_removed", metadata.channel_id)
+ }
+ };
+ let entity_id = channel_id.to_string();
- async fn handle_channel_participant_added(
- &self,
- event_id: Uuid,
- metadata: ChannelParticipantAddedMetadata,
- ) -> Result<(), WebhookEventIngestionError> {
- tracing::debug!(
- %event_id,
- channel_id = %metadata.channel_id,
- "channel.participant_added: webhook ingestion not implemented yet"
- );
- Ok(())
- }
+ let broker_envelope = serde_json::to_value(event)?;
+ Ok(normalized_event(
+ event.event_id,
+ event.schema_version,
+ event_name,
+ CHANNEL_ENTITY_TYPE,
+ &entity_id,
+ broker_envelope,
+ ))
+}
- async fn handle_channel_participant_removed(
- &self,
- event_id: Uuid,
- metadata: ChannelParticipantRemovedMetadata,
- ) -> Result<(), WebhookEventIngestionError> {
- tracing::debug!(
- %event_id,
- channel_id = %metadata.channel_id,
- "channel.participant_removed: webhook ingestion not implemented yet"
- );
- Ok(())
+fn normalized_event(
+ event_id: Uuid,
+ schema_version: u8,
+ event_name: &str,
+ entity_type: &str,
+ entity_id: &str,
+ broker_envelope: serde_json::Value,
+) -> NormalizedWebhookEvent {
+ NormalizedWebhookEvent {
+ event_id: event_id.to_string(),
+ schema_version,
+ event_name: event_name.to_string(),
+ entity_type: entity_type.to_string(),
+ entity_id: entity_id.to_string(),
+ ordering_key: entity_id.to_string(),
+ occurred_at: Utc::now(),
+ broker_envelope,
}
}
-impl<A: EntityAccessService> WebhookEventIngestionService for WebhookEventIngestionServiceImpl<A> {
+impl<A, R, Q> WebhookEventIngestionService for WebhookEventIngestionServiceImpl<A, R, Q>
+where
+ A: EntityAccessService,
+ R: WebhookRepo + WebhookWorkspaceResolver,
+ Q: WebhookEventEnqueuer,
+{
#[tracing::instrument(skip(self, event), fields(event_id = %event.event_id), err)]
async fn ingest_document_event(
&self,
event: Event<DocumentTopicEvent>,
) -> Result<(), WebhookEventIngestionError> {
- tracing::trace!(?event, "ingesting document event");
- let event_id = event.event_id;
- match event.event {
- DocumentTopicEvent::Created(metadata) => {
- self.handle_document_created(event_id, metadata).await
- }
- DocumentTopicEvent::Updated(metadata) => {
- self.handle_document_updated(event_id, metadata).await
- }
- DocumentTopicEvent::Deleted(metadata) => {
- self.handle_document_deleted(event_id, metadata).await
- }
- DocumentTopicEvent::Copied(metadata) => {
- self.handle_document_copied(event_id, metadata).await
- }
- }
+ let event = normalized_document_event(&event)?;
+ self.ingest_normalized_event(event, EntityType::Document)
+ .await
}
#[tracing::instrument(skip(self, event), fields(event_id = %event.event_id), err)]
@@ -338,45 +304,8 @@ impl<A: EntityAccessService> WebhookEventIngestionService for WebhookEventIngest
&self,
event: Event<ChannelTopicEvent>,
) -> Result<(), WebhookEventIngestionError> {
- tracing::trace!(?event, "ingesting channel event");
- let event_id = event.event_id;
- match event.event {
- ChannelTopicEvent::Created(metadata) => {
- self.handle_channel_created(event_id, metadata).await
- }
- ChannelTopicEvent::Updated(metadata) => {
- self.handle_channel_updated(event_id, metadata).await
- }
- ChannelTopicEvent::Deleted(metadata) => {
- self.handle_channel_deleted(event_id, metadata).await
- }
- ChannelTopicEvent::MessagePosted(metadata) => {
- self.handle_channel_message_posted(event_id, metadata).await
- }
- ChannelTopicEvent::MessagePatched(metadata) => {
- self.handle_channel_message_patched(event_id, metadata)
- .await
- }
- ChannelTopicEvent::MessageDeleted(metadata) => {
- self.handle_channel_message_deleted(event_id, metadata)
- .await
- }
- ChannelTopicEvent::MessageAttachmentCreated(metadata) => {
- self.handle_channel_message_attachment_created(event_id, metadata)
- .await
- }
- ChannelTopicEvent::MessageAttachmentRemoved(metadata) => {
- self.handle_channel_message_attachment_removed(event_id, metadata)
- .await
- }
- ChannelTopicEvent::ParticipantAdded(metadata) => {
- self.handle_channel_participant_added(event_id, metadata)
- .await
- }
- ChannelTopicEvent::ParticipantRemoved(metadata) => {
- self.handle_channel_participant_removed(event_id, metadata)
- .await
- }
- }
+ let event = normalized_channel_event(&event)?;
+ self.ingest_normalized_event(event, EntityType::Channel)
+ .await
}
}
diff --git a/rust/cloud-storage/webhook/src/domain/ingestion/test.rs b/rust/cloud-storage/webhook/src/domain/ingestion/test.rs
index 955bc25612..a75a8b2db4 100644
--- a/rust/cloud-storage/webhook/src/domain/ingestion/test.rs
+++ b/rust/cloud-storage/webhook/src/domain/ingestion/test.rs
@@ -1,12 +1,45 @@
use super::*;
+use crate::domain::{
+ models::{
+ CreateWebhookRequest, PatchWebhookRequest, Webhook, WebhookEventQueueMessage, WebhookStatus,
+ },
+ ports::{WebhookEventEnqueuer, WebhookRepo, WebhookWorkspaceResolver},
+};
use channel_sender::ChannelSender;
-use channels::domain::models::ChannelType;
+use channels::domain::{
+ broker_events::{
+ ChannelCreatedMetadata, ChannelDeletedMetadata, ChannelMessageAttachmentCreatedMetadata,
+ ChannelMessageAttachmentRemovedMetadata, ChannelMessageDeletedMetadata,
+ ChannelMessagePatchedMetadata, ChannelMessagePostedMetadata,
+ ChannelParticipantAddedMetadata, ChannelParticipantRemovedMetadata, ChannelUpdatedMetadata,
+ },
+ models::ChannelType,
+};
use chrono::{DateTime, Utc};
+use documents::domain::events::{
+ DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata,
+ DocumentUpdatedMetadata,
+};
use entity_access::domain::models::{
AccessLevel, CallChannelInfo, EntityAccessReceipt, EntityPermission, RequiredPermission,
UserTeamInfo,
};
use macro_user_id::{lowercased::Lowercase, user_id::MacroUserId};
+use serde_json::Value;
+use std::{
+ collections::BTreeMap,
+ sync::{Arc, Mutex, MutexGuard},
+ time::Duration,
+};
+
+const DOCUMENT_ID: &str = "11111111-1111-1111-1111-111111111111";
+const COPIED_DOCUMENT_ID: &str = "22222222-2222-2222-2222-222222222222";
+const PERSONAL_WORKSPACE_ID: &str = "macro|[email protected]";
+const TEAM_WORKSPACE_ID: &str = "33333333-3333-3333-3333-333333333333";
+
+fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
+ mutex.lock().expect("test mutex is not poisoned")
+}
fn user_id(id: &str) -> MacroUserIdStr<'static> {
MacroUserIdStr::try_from(id.to_string()).expect("valid user id")
@@ -22,10 +55,27 @@ fn timestamp() -> DateTime<Utc> {
.with_timezone(&Utc)
}
-/// Mock access service: only [`EntityAccessService::get_users_by_entity`] is
-/// exercised by the ingestion service.
+#[derive(Debug, Clone, Copy)]
+enum AccessFailure {
+ Internal,
+}
+
#[derive(Clone)]
-struct MockAccessService;
+struct MockAccessService {
+ users: Vec<MacroUserIdStr<'static>>,
+ failure: Option<AccessFailure>,
+ calls: Arc<Mutex<Vec<(String, EntityType)>>>,
+}
+
+impl MockAccessService {
+ fn with_users(users: Vec<MacroUserIdStr<'static>>) -> Self {
+ Self {
+ users,
+ failure: None,
+ calls: Arc::default(),
+ }
+ }
+}
impl EntityAccessService for MockAccessService {
async fn generate_entity_access_receipt<T: RequiredPermission>(
@@ -87,10 +137,14 @@ impl EntityAccessService for MockAccessService {
async fn get_users_by_entity(
&self,
- _entity_id: &str,
- _entity_type: EntityType,
+ entity_id: &str,
+ entity_type: EntityType,
) -> Result<Vec<MacroUserIdStr<'static>>, AccessError> {
- Ok(vec![user_id("macro|[email protected]")])
+ lock(&self.calls).push((entity_id.to_string(), entity_type));
+ match self.failure {
+ Some(AccessFailure::Internal) => Err(AccessError::Internal),
+ None => Ok(self.users.clone()),
+ }
}
async fn get_call_channel(
@@ -115,166 +169,749 @@ impl EntityAccessService for MockAccessService {
}
}
-fn service() -> WebhookEventIngestionServiceImpl<MockAccessService> {
- WebhookEventIngestionServiceImpl::new(Arc::new(MockAccessService))
+#[derive(Debug, Clone, PartialEq, Eq)]
+struct MatchCall {
+ workspace_ids: Vec<String>,
+ event_name: String,
+ entity_id: String,
+}
+
+struct MockRepositoryState {
+ resolved_workspace_ids: Vec<String>,
+ matching_webhooks: Vec<Webhook>,
+ fail_workspace_resolution: bool,
+ fail_matching: bool,
+ workspace_calls: Vec<Vec<MacroUserIdStr<'static>>>,
+ match_calls: Vec<MatchCall>,
+}
+
+#[derive(Clone)]
+struct MockRepository {
+ state: Arc<Mutex<MockRepositoryState>>,
+}
+
+impl MockRepository {
+ fn new(resolved_workspace_ids: Vec<String>, matching_webhooks: Vec<Webhook>) -> Self {
+ Self {
+ state: Arc::new(Mutex::new(MockRepositoryState {
+ resolved_workspace_ids,
+ matching_webhooks,
+ fail_workspace_resolution: false,
+ fail_matching: false,
+ workspace_calls: Vec::new(),
+ match_calls: Vec::new(),
+ })),
+ }
+ }
+}
+
+impl WebhookWorkspaceResolver for MockRepository {
+ type Err = anyhow::Error;
+
+ async fn resolve_workspace_ids(
+ &self,
+ people: Vec<MacroUserIdStr<'static>>,
+ ) -> Result<Vec<String>, Self::Err> {
+ let mut state = lock(&self.state);
+ state.workspace_calls.push(people);
+ if state.fail_workspace_resolution {
+ anyhow::bail!("workspace resolver unavailable");
+ }
+ Ok(state.resolved_workspace_ids.clone())
+ }
+}
+
+impl WebhookRepo for MockRepository {
+ type Err = anyhow::Error;
+
+ async fn create_webhook(
+ &self,
+ _created_by_user_id: MacroUserIdStr<'static>,
+ _workspace_id: String,
+ _request: CreateWebhookRequest,
+ _signing_secret: String,
+ _headers: Value,
+ ) -> Result<Webhook, Self::Err> {
+ unimplemented!("not used by webhook event ingestion")
+ }
+
+ async fn get_webhook(&self, _webhook_id: String) -> Result<Option<Webhook>, Self::Err> {
+ unimplemented!("not used by webhook event ingestion")
+ }
+
+ async fn list_active_webhooks_matching_event(
+ &self,
+ workspace_ids: Vec<String>,
+ event_name: String,
+ entity_id: String,
+ ) -> Result<Vec<Webhook>, Self::Err> {
+ let mut state = lock(&self.state);
+ state.match_calls.push(MatchCall {
+ workspace_ids,
+ event_name,
+ entity_id,
+ });
+ if state.fail_matching {
+ anyhow::bail!("webhook repository unavailable");
+ }
+ Ok(state.matching_webhooks.clone())
+ }
+
+ async fn patch_webhook(
+ &self,
+ _webhook_id: String,
+ _request: PatchWebhookRequest,
+ ) -> Result<Option<Webhook>, Self::Err> {
+ unimplemented!("not used by webhook event ingestion")
+ }
+
+ async fn delete_webhook(&self, _webhook_id: String) -> Result<Option<Webhook>, Self::Err> {
+ unimplemented!("not used by webhook event ingestion")
+ }
+
+ async fn set_webhook_validity(
+ &self,
+ _webhook_id: String,
+ _is_valid: bool,
+ ) -> Result<Option<Webhook>, Self::Err> {
+ unimplemented!("not used by webhook event ingestion")
+ }
+
+ async fn get_user_team_workspace_id(
+ &self,
+ _user_id: MacroUserIdStr<'static>,
+ ) -> Result<Option<String>, Self::Err> {
+ unimplemented!("not used by webhook event ingestion")
+ }
+}
+
+#[derive(Default)]
+struct MockEnqueuerState {
+ attempted_messages: Vec<WebhookEventQueueMessage>,
+ completed_webhook_ids: Vec<String>,
+ failing_webhook_id: Option<String>,
+ delayed_webhook_id: Option<String>,
+}
+
+#[derive(Clone, Default)]
+struct MockEnqueuer {
+ state: Arc<Mutex<MockEnqueuerState>>,
+}
+
+impl WebhookEventEnqueuer for MockEnqueuer {
+ type Err = anyhow::Error;
+
+ async fn enqueue(&self, message: WebhookEventQueueMessage) -> Result<(), Self::Err> {
+ let (should_fail, should_delay) = {
+ let mut state = lock(&self.state);
+ let should_fail = state.failing_webhook_id.as_ref() == Some(&message.webhook_id);
+ let should_delay = state.delayed_webhook_id.as_ref() == Some(&message.webhook_id);
+ state.attempted_messages.push(message.clone());
+ (should_fail, should_delay)
+ };
+
+ if should_delay {
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ lock(&self.state)
+ .completed_webhook_ids
+ .push(message.webhook_id.clone());
+
+ if should_fail {
+ anyhow::bail!("queue unavailable");
+ }
+ Ok(())
+ }
+}
+
+type TestService =
+ WebhookEventIngestionServiceImpl<MockAccessService, MockRepository, MockEnqueuer>;
+
+fn service(
+ access: MockAccessService,
+ repository: MockRepository,
+ enqueuer: MockEnqueuer,
+) -> TestService {
+ WebhookEventIngestionServiceImpl::new(Arc::new(access), repository, enqueuer)
+}
+
+fn webhook(id: &str, workspace_id: &str) -> Webhook {
+ Webhook {
+ id: id.to_string(),
+ workspace_id: workspace_id.to_string(),
+ name: id.to_string(),
+ endpoint_url: "https://example.com/webhook".to_string(),
+ signing_secret: "not-queued".to_string(),
+ headers: BTreeMap::from([("X-Custom".to_string(), "not-queued".to_string())]),
+ status: WebhookStatus::Active,
+ is_valid: true,
+ created_by_user_id: PERSONAL_WORKSPACE_ID.to_string(),
+ created_at: timestamp(),
+ updated_at: timestamp(),
+ deleted_at: None,
+ filters: Vec::new(),
+ }
+}
+
+#[derive(Clone)]
+enum TestBrokerEvent {
+ Document(Event<DocumentTopicEvent>),
+ Channel(Event<ChannelTopicEvent>),
}
-fn document_events() -> Vec<DocumentTopicEvent> {
+impl TestBrokerEvent {
+ async fn ingest(&self, service: &TestService) -> Result<(), WebhookEventIngestionError> {
+ match self {
+ Self::Document(event) => service.ingest_document_event(event.clone()).await,
+ Self::Channel(event) => service.ingest_channel_event(event.clone()).await,
+ }
+ }
+
+ fn envelope(&self) -> Value {
+ match self {
+ Self::Document(event) => serde_json::to_value(event).expect("serializable event"),
+ Self::Channel(event) => serde_json::to_value(event).expect("serializable event"),
+ }
+ }
+
+ fn event_id(&self) -> Uuid {
+ match self {
+ Self::Document(event) => event.event_id,
+ Self::Channel(event) => event.event_id,
+ }
+ }
+
+ fn schema_version(&self) -> u8 {
+ match self {
+ Self::Document(event) => event.schema_version,
+ Self::Channel(event) => event.schema_version,
+ }
+ }
+}
+
+struct EventCase {
+ event: TestBrokerEvent,
+ event_name: &'static str,
+ entity_type: EntityType,
+ normalized_entity_type: &'static str,
+ entity_id: String,
+}
+
+fn document_event_cases() -> Vec<EventCase> {
vec![
- DocumentTopicEvent::Created(DocumentCreatedMetadata {
- document_id: "doc_1".to_string(),
- owner: user_id("macro|[email protected]"),
- document_name: "notes".to_string(),
- file_type: None,
- project_id: None,
- sub_type: None,
- created_at: Some(timestamp()),
- }),
- DocumentTopicEvent::Updated(DocumentUpdatedMetadata {
- document_id: "doc_1".to_string(),
- owner: user_id("macro|[email protected]"),
- actor_user_id: Some(user_id("macro|[email protected]")),
- document_name: Some("renamed".to_string()),
- previous_project_id: None,
- project_id: None,
- file_type: None,
- share_permission_updated: false,
- }),
- DocumentTopicEvent::Deleted(DocumentDeletedMetadata {
- document_id: "doc_1".to_string(),
- actor_user_id: Some(user_id("macro|[email protected]")),
- project_id: None,
- }),
- DocumentTopicEvent::Copied(DocumentCopiedMetadata {
- document_id: "doc_2".to_string(),
- source_document_id: "doc_1".to_string(),
- source_version_id: None,
- owner: user_id("macro|[email protected]"),
- document_name: "notes (copy)".to_string(),
- file_type: None,
- project_id: None,
- sub_type: None,
- }),
+ EventCase {
+ event: TestBrokerEvent::Document(Event::with_schema_version(
+ DocumentTopicEvent::Created(DocumentCreatedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ owner: user_id("macro|[email protected]"),
+ document_name: "notes".to_string(),
+ file_type: None,
+ project_id: None,
+ sub_type: None,
+ created_at: Some(timestamp()),
+ }),
+ 2,
+ )),
+ event_name: "document.created",
+ entity_type: EntityType::Document,
+ normalized_entity_type: DOCUMENT_ENTITY_TYPE,
+ entity_id: DOCUMENT_ID.to_string(),
+ },
+ EventCase {
+ event: TestBrokerEvent::Document(Event::with_schema_version(
+ DocumentTopicEvent::Updated(DocumentUpdatedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ owner: user_id("macro|[email protected]"),
+ actor_user_id: Some(user_id("macro|[email protected]")),
+ document_name: Some("renamed".to_string()),
+ previous_project_id: None,
+ project_id: None,
+ file_type: None,
+ share_permission_updated: false,
+ }),
+ 2,
+ )),
+ event_name: "document.updated",
+ entity_type: EntityType::Document,
+ normalized_entity_type: DOCUMENT_ENTITY_TYPE,
+ entity_id: DOCUMENT_ID.to_string(),
+ },
+ EventCase {
+ event: TestBrokerEvent::Document(Event::with_schema_version(
+ DocumentTopicEvent::Deleted(DocumentDeletedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ actor_user_id: Some(user_id("macro|[email protected]")),
+ project_id: None,
+ }),
+ 2,
+ )),
+ event_name: "document.deleted",
+ entity_type: EntityType::Document,
+ normalized_entity_type: DOCUMENT_ENTITY_TYPE,
+ entity_id: DOCUMENT_ID.to_string(),
+ },
+ EventCase {
+ event: TestBrokerEvent::Document(Event::with_schema_version(
+ DocumentTopicEvent::Copied(DocumentCopiedMetadata {
+ document_id: COPIED_DOCUMENT_ID.to_string(),
+ source_document_id: DOCUMENT_ID.to_string(),
+ source_version_id: None,
+ owner: user_id("macro|[email protected]"),
+ document_name: "notes (copy)".to_string(),
+ file_type: None,
+ project_id: None,
+ sub_type: None,
+ }),
+ 2,
+ )),
+ event_name: "document.copied",
+ entity_type: EntityType::Document,
+ normalized_entity_type: DOCUMENT_ENTITY_TYPE,
+ entity_id: COPIED_DOCUMENT_ID.to_string(),
+ },
]
}
-fn channel_events() -> Vec<ChannelTopicEvent> {
- let channel_id = Uuid::nil();
- let message_id = Uuid::nil();
+fn channel_event_cases() -> Vec<EventCase> {
+ let channel_id = Uuid::parse_str("44444444-4444-4444-4444-444444444444").unwrap();
+ let message_id = Uuid::parse_str("55555555-5555-5555-5555-555555555555").unwrap();
+ let owner = "macro|[email protected]";
+ let member = "macro|[email protected]";
+
vec![
- ChannelTopicEvent::Created(ChannelCreatedMetadata {
- channel_id,
- actor: sender("macro|[email protected]"),
- channel_type: ChannelType::Team,
- channel_name: Some("general".to_string()),
- participant_user_ids: vec![user_id("macro|[email protected]")],
- }),
- ChannelTopicEvent::Updated(ChannelUpdatedMetadata {
- channel_id,
- actor: user_id("macro|[email protected]"),
- previous_name: Some("general".to_string()),
- channel_name: Some("announcements".to_string()),
- }),
- ChannelTopicEvent::Deleted(ChannelDeletedMetadata {
- channel_id,
- actor: sender("macro|[email protected]"),
- }),
- ChannelTopicEvent::MessagePosted(ChannelMessagePostedMetadata {
- channel_id,
- message_id,
- thread_id: None,
- sender: sender("macro|[email protected]"),
- triggered_by: None,
- channel_type: ChannelType::Team,
- content: "hello".to_string(),
- mentions: vec![],
- attachments: vec![],
- created_at: timestamp(),
- }),
- ChannelTopicEvent::MessagePatched(ChannelMessagePatchedMetadata {
- channel_id,
- message_id,
- thread_id: None,
- actor: sender("macro|[email protected]"),
- content: "hello (edited)".to_string(),
- edited_at: Some(timestamp()),
- updated_at: timestamp(),
- }),
- ChannelTopicEvent::MessageDeleted(ChannelMessageDeletedMetadata {
- channel_id,
- message_id,
- thread_id: None,
- actor: sender("macro|[email protected]"),
- deleted_at: Some(timestamp()),
- }),
- ChannelTopicEvent::MessageAttachmentCreated(ChannelMessageAttachmentCreatedMetadata {
- channel_id,
- message_id,
- actor: sender("macro|[email protected]"),
- attachments: vec![],
- }),
- ChannelTopicEvent::MessageAttachmentRemoved(ChannelMessageAttachmentRemovedMetadata {
- channel_id,
- message_id,
- actor: sender("macro|[email protected]"),
- attachments: vec![],
- }),
- ChannelTopicEvent::ParticipantAdded(ChannelParticipantAddedMetadata {
- channel_id,
- channel_type: ChannelType::Team,
- added_by: sender("macro|[email protected]"),
- added_user_ids: vec![user_id("macro|[email protected]")],
- }),
- ChannelTopicEvent::ParticipantRemoved(ChannelParticipantRemovedMetadata {
- channel_id,
- channel_type: ChannelType::Team,
- removed_by: user_id("macro|[email protected]"),
- removed_user_ids: vec![user_id("macro|[email protected]")],
- }),
+ EventCase {
+ event: TestBrokerEvent::Channel(Event::with_schema_version(
+ ChannelTopicEvent::Created(ChannelCreatedMetadata {
+ channel_id,
+ actor: sender(owner),
+ channel_type: ChannelType::Team,
+ channel_name: Some("general".to_string()),
+ participant_user_ids: vec![user_id(owner)],
+ }),
+ 3,
+ )),
+ event_name: "channel.created",
+ entity_type: EntityType::Channel,
+ normalized_entity_type: CHANNEL_ENTITY_TYPE,
+ entity_id: channel_id.to_string(),
+ },
+ EventCase {
+ event: TestBrokerEvent::Channel(Event::with_schema_version(
+ ChannelTopicEvent::Updated(ChannelUpdatedMetadata {
+ channel_id,
+ actor: user_id(owner),
+ previous_name: Some("general".to_string()),
+ channel_name: Some("announcements".to_string()),
+ }),
+ 3,
+ )),
+ event_name: "channel.updated",
+ entity_type: EntityType::Channel,
+ normalized_entity_type: CHANNEL_ENTITY_TYPE,
+ entity_id: channel_id.to_string(),
+ },
+ EventCase {
+ event: TestBrokerEvent::Channel(Event::with_schema_version(
+ ChannelTopicEvent::Deleted(ChannelDeletedMetadata {
+ channel_id,
+ actor: sender(owner),
+ }),
+ 3,
+ )),
+ event_name: "channel.deleted",
+ entity_type: EntityType::Channel,
+ normalized_entity_type: CHANNEL_ENTITY_TYPE,
+ entity_id: channel_id.to_string(),
+ },
+ EventCase {
+ event: TestBrokerEvent::Channel(Event::with_schema_version(
+ ChannelTopicEvent::MessagePosted(ChannelMessagePostedMetadata {
+ channel_id,
+ message_id,
+ thread_id: None,
+ sender: sender(member),
+ triggered_by: None,
+ channel_type: ChannelType::Team,
+ content: "hello".to_string(),
+ mentions: vec![],
+ attachments: vec![],
+ created_at: timestamp(),
+ }),
+ 3,
+ )),
+ event_name: "channel.message_posted",
+ entity_type: EntityType::Channel,
+ normalized_entity_type: CHANNEL_ENTITY_TYPE,
+ entity_id: channel_id.to_string(),
+ },
+ EventCase {
+ event: TestBrokerEvent::Channel(Event::with_schema_version(
+ ChannelTopicEvent::MessagePatched(ChannelMessagePatchedMetadata {
+ channel_id,
+ message_id,
+ thread_id: None,
+ actor: sender(member),
+ content: "hello (edited)".to_string(),
+ edited_at: Some(timestamp()),
+ updated_at: timestamp(),
+ }),
+ 3,
+ )),
+ event_name: "channel.message_patched",
+ entity_type: EntityType::Channel,
+ normalized_entity_type: CHANNEL_ENTITY_TYPE,
+ entity_id: channel_id.to_string(),
+ },
+ EventCase {
+ event: TestBrokerEvent::Channel(Event::with_schema_version(
+ ChannelTopicEvent::MessageDeleted(ChannelMessageDeletedMetadata {
+ channel_id,
+ message_id,
+ thread_id: None,
+ actor: sender(member),
+ deleted_at: Some(timestamp()),
+ }),
+ 3,
+ )),
+ event_name: "channel.message_deleted",
+ entity_type: EntityType::Channel,
+ normalized_entity_type: CHANNEL_ENTITY_TYPE,
+ entity_id: channel_id.to_string(),
+ },
+ EventCase {
+ event: TestBrokerEvent::Channel(Event::with_schema_version(
+ ChannelTopicEvent::MessageAttachmentCreated(
+ ChannelMessageAttachmentCreatedMetadata {
+ channel_id,
+ message_id,
+ actor: sender(member),
+ attachments: vec![],
+ },
+ ),
+ 3,
+ )),
+ event_name: "channel.message_attachment_created",
+ entity_type: EntityType::Channel,
+ normalized_entity_type: CHANNEL_ENTITY_TYPE,
+ entity_id: channel_id.to_string(),
+ },
+ EventCase {
+ event: TestBrokerEvent::Channel(Event::with_schema_version(
+ ChannelTopicEvent::MessageAttachmentRemoved(
+ ChannelMessageAttachmentRemovedMetadata {
+ channel_id,
+ message_id,
+ actor: sender(member),
+ attachments: vec![],
+ },
+ ),
+ 3,
+ )),
+ event_name: "channel.message_attachment_removed",
+ entity_type: EntityType::Channel,
+ normalized_entity_type: CHANNEL_ENTITY_TYPE,
+ entity_id: channel_id.to_string(),
+ },
+ EventCase {
+ event: TestBrokerEvent::Channel(Event::with_schema_version(
+ ChannelTopicEvent::ParticipantAdded(ChannelParticipantAddedMetadata {
+ channel_id,
+ channel_type: ChannelType::Team,
+ added_by: sender(owner),
+ added_user_ids: vec![user_id(member)],
+ }),
+ 3,
+ )),
+ event_name: "channel.participant_added",
+ entity_type: EntityType::Channel,
+ normalized_entity_type: CHANNEL_ENTITY_TYPE,
+ entity_id: channel_id.to_string(),
+ },
+ EventCase {
+ event: TestBrokerEvent::Channel(Event::with_schema_version(
+ ChannelTopicEvent::ParticipantRemoved(ChannelParticipantRemovedMetadata {
+ channel_id,
+ channel_type: ChannelType::Team,
+ removed_by: user_id(owner),
+ removed_user_ids: vec![user_id(member)],
+ }),
+ 3,
+ )),
+ event_name: "channel.participant_removed",
+ entity_type: EntityType::Channel,
+ normalized_entity_type: CHANNEL_ENTITY_TYPE,
+ entity_id: channel_id.to_string(),
+ },
]
}
#[tokio::test]
-async fn ingests_every_document_event_variant() {
- let service = service();
- for event in document_events() {
- service
- .ingest_document_event(Event::new(event))
+async fn normalizes_and_matches_all_fourteen_event_variants() {
+ let event_cases = document_event_cases()
+ .into_iter()
+ .chain(channel_event_cases())
+ .collect::<Vec<_>>();
+ assert_eq!(event_cases.len(), 14);
+
+ for event_case in event_cases {
+ let access = MockAccessService::with_users(vec![user_id(PERSONAL_WORKSPACE_ID)]);
+ let repository = MockRepository::new(
+ vec![PERSONAL_WORKSPACE_ID.to_string()],
+ vec![webhook("wh_match", PERSONAL_WORKSPACE_ID)],
+ );
+ let enqueuer = MockEnqueuer::default();
+ let service = service(access.clone(), repository.clone(), enqueuer.clone());
+ let expected_envelope = event_case.event.envelope();
+ let before_ingestion = Utc::now();
+
+ event_case
+ .event
+ .ingest(&service)
.await
- .expect("stub handlers always succeed");
+ .unwrap_or_else(|error| {
+ panic!("{} should be ingested: {error}", event_case.event_name)
+ });
+ let after_ingestion = Utc::now();
+
+ assert_eq!(
+ lock(&access.calls).as_slice(),
+ &[(event_case.entity_id.clone(), event_case.entity_type)],
+ "entity access mapping for {}",
+ event_case.event_name
+ );
+ let repository_state = lock(&repository.state);
+ assert_eq!(
+ repository_state.match_calls.as_slice(),
+ &[MatchCall {
+ workspace_ids: vec![PERSONAL_WORKSPACE_ID.to_string()],
+ event_name: event_case.event_name.to_string(),
+ entity_id: event_case.entity_id.clone(),
+ }],
+ "repository matching for {}",
+ event_case.event_name
+ );
+ drop(repository_state);
+
+ let enqueuer_state = lock(&enqueuer.state);
+ assert_eq!(enqueuer_state.attempted_messages.len(), 1);
+ let message = &enqueuer_state.attempted_messages[0];
+ assert_eq!(message.webhook_id, "wh_match");
+ assert_eq!(
+ message.event.event_id,
+ event_case.event.event_id().to_string()
+ );
+ assert_eq!(
+ message.event.schema_version,
+ event_case.event.schema_version()
+ );
+ assert_eq!(message.event.event_name, event_case.event_name);
+ assert_eq!(message.event.entity_type, event_case.normalized_entity_type);
+ assert_eq!(message.event.entity_id, event_case.entity_id);
+ assert_eq!(message.event.ordering_key, event_case.entity_id);
+ assert_eq!(message.event.broker_envelope, expected_envelope);
+ assert_eq!(
+ message.event.broker_envelope["event_type"],
+ event_case.event_name
+ );
+ assert!(message.event.occurred_at >= before_ingestion);
+ assert!(message.event.occurred_at <= after_ingestion);
}
}
#[tokio::test]
-async fn ingests_every_channel_event_variant() {
- let service = service();
- for event in channel_events() {
- service
- .ingest_channel_event(Event::new(event))
- .await
- .expect("stub handlers always succeed");
+async fn no_accessors_resolves_no_workspaces_and_enqueues_nothing() {
+ let access = MockAccessService::with_users(Vec::new());
+ let repository = MockRepository::new(Vec::new(), Vec::new());
+ let enqueuer = MockEnqueuer::default();
+ let service = service(access, repository.clone(), enqueuer.clone());
+ let event = document_event_cases().remove(0).event;
+
+ event.ingest(&service).await.expect("no access is a no-op");
+
+ let repository_state = lock(&repository.state);
+ assert_eq!(repository_state.workspace_calls, vec![Vec::new()]);
+ assert_eq!(
+ repository_state.match_calls,
+ vec![MatchCall {
+ workspace_ids: Vec::new(),
+ event_name: "document.created".to_string(),
+ entity_id: DOCUMENT_ID.to_string(),
+ }]
+ );
+ assert!(lock(&enqueuer.state).attempted_messages.is_empty());
+}
+
+#[tokio::test]
+async fn no_matching_webhooks_enqueues_nothing_after_workspace_expansion() {
+ let reader = user_id(PERSONAL_WORKSPACE_ID);
+ let access = MockAccessService::with_users(vec![reader.clone()]);
+ let repository = MockRepository::new(
+ vec![
+ PERSONAL_WORKSPACE_ID.to_string(),
+ TEAM_WORKSPACE_ID.to_string(),
+ ],
+ Vec::new(),
+ );
+ let enqueuer = MockEnqueuer::default();
+ let service = service(access, repository.clone(), enqueuer.clone());
+ let event = document_event_cases().remove(0).event;
+
+ event.ingest(&service).await.expect("no matches is a no-op");
+
+ let repository_state = lock(&repository.state);
+ assert_eq!(repository_state.workspace_calls, vec![vec![reader]]);
+ assert_eq!(
+ repository_state.match_calls[0].workspace_ids,
+ vec![
+ PERSONAL_WORKSPACE_ID.to_string(),
+ TEAM_WORKSPACE_ID.to_string(),
+ ]
+ );
+ assert!(lock(&enqueuer.state).attempted_messages.is_empty());
+}
+
+#[tokio::test]
+async fn fans_out_exact_envelope_once_per_personal_and_team_webhook() {
+ let event = document_event_cases().remove(3).event;
+ let expected_envelope = event.envelope();
+ let reader = user_id(PERSONAL_WORKSPACE_ID);
+ let access = MockAccessService::with_users(vec![reader.clone()]);
+ let repository = MockRepository::new(
+ vec![
+ PERSONAL_WORKSPACE_ID.to_string(),
+ TEAM_WORKSPACE_ID.to_string(),
+ ],
+ vec![
+ webhook("wh_personal", PERSONAL_WORKSPACE_ID),
+ webhook("wh_team", TEAM_WORKSPACE_ID),
+ ],
+ );
+ let enqueuer = MockEnqueuer::default();
+ let service = service(access, repository.clone(), enqueuer.clone());
+
+ event.ingest(&service).await.expect("fan-out succeeds");
+
+ let repository_state = lock(&repository.state);
+ assert_eq!(repository_state.workspace_calls, vec![vec![reader]]);
+ assert_eq!(
+ repository_state.match_calls,
+ vec![MatchCall {
+ workspace_ids: vec![
+ PERSONAL_WORKSPACE_ID.to_string(),
+ TEAM_WORKSPACE_ID.to_string(),
+ ],
+ event_name: "document.copied".to_string(),
+ entity_id: COPIED_DOCUMENT_ID.to_string(),
+ }]
+ );
+ drop(repository_state);
+
+ let enqueuer_state = lock(&enqueuer.state);
+ let mut webhook_ids = enqueuer_state
+ .attempted_messages
+ .iter()
+ .map(|message| message.webhook_id.as_str())
+ .collect::<Vec<_>>();
+ webhook_ids.sort_unstable();
+ assert_eq!(webhook_ids, vec!["wh_personal", "wh_team"]);
+ for message in &enqueuer_state.attempted_messages {
+ assert_eq!(message.event.broker_envelope, expected_envelope);
+ assert_eq!(message.event.entity_id, COPIED_DOCUMENT_ID);
+ }
+}
+
+#[tokio::test]
+async fn enqueue_failure_is_transient_after_every_send_completes() {
+ let access = MockAccessService::with_users(vec![user_id(PERSONAL_WORKSPACE_ID)]);
+ let repository = MockRepository::new(
+ vec![PERSONAL_WORKSPACE_ID.to_string()],
+ vec![
+ webhook("wh_fail", PERSONAL_WORKSPACE_ID),
+ webhook("wh_slow", PERSONAL_WORKSPACE_ID),
+ ],
+ );
+ let enqueuer = MockEnqueuer::default();
+ {
+ let mut state = lock(&enqueuer.state);
+ state.failing_webhook_id = Some("wh_fail".to_string());
+ state.delayed_webhook_id = Some("wh_slow".to_string());
}
+ let service = service(access, repository, enqueuer.clone());
+ let event = document_event_cases().remove(0).event;
+
+ let error = event
+ .ingest(&service)
+ .await
+ .expect_err("one failed send fails ingestion");
+
+ assert!(matches!(error, WebhookEventIngestionError::Enqueue(_)));
+ assert!(error.is_transient());
+ let mut completed_webhook_ids = lock(&enqueuer.state).completed_webhook_ids.clone();
+ completed_webhook_ids.sort_unstable();
+ assert_eq!(completed_webhook_ids, vec!["wh_fail", "wh_slow"]);
+}
+
+#[tokio::test]
+async fn malformed_document_id_is_permanent_and_skips_access_resolution() {
+ let access = MockAccessService::with_users(vec![user_id(PERSONAL_WORKSPACE_ID)]);
+ let repository = MockRepository::new(Vec::new(), Vec::new());
+ let enqueuer = MockEnqueuer::default();
+ let service = service(access.clone(), repository, enqueuer);
+ let event = Event::new(DocumentTopicEvent::Deleted(DocumentDeletedMetadata {
+ document_id: "not-a-uuid".to_string(),
+ actor_user_id: None,
+ project_id: None,
+ }));
+
+ let error = service
+ .ingest_document_event(event)
+ .await
+ .expect_err("malformed ids are rejected");
+
+ assert!(matches!(
+ error,
+ WebhookEventIngestionError::InvalidEntityId { .. }
+ ));
+ assert!(!error.is_transient());
+ assert!(lock(&access.calls).is_empty());
}
#[test]
-fn classifies_transient_errors() {
+fn classifies_adapter_and_contract_errors() {
let database_error = WebhookEventIngestionError::EntityAccess(AccessError::DatabaseError(
sqlx::Error::PoolTimedOut,
));
assert!(database_error.is_transient());
+ assert!(WebhookEventIngestionError::EntityAccess(AccessError::Internal).is_transient());
+ assert!(!WebhookEventIngestionError::EntityAccess(AccessError::Unauthorized).is_transient());
- let unauthorized = WebhookEventIngestionError::EntityAccess(AccessError::Unauthorized);
- assert!(!unauthorized.is_transient());
-
- let internal = WebhookEventIngestionError::Internal(anyhow::anyhow!("adapter bug"));
- assert!(!internal.is_transient());
+ let serialization_error = serde_json::from_str::<Value>("{").expect_err("invalid json");
+ assert!(!WebhookEventIngestionError::Serialization(serialization_error).is_transient());
+ assert!(WebhookEventIngestionError::WorkspaceResolution(anyhow::anyhow!("db")).is_transient());
+ assert!(WebhookEventIngestionError::Repository(anyhow::anyhow!("db")).is_transient());
+ assert!(WebhookEventIngestionError::Enqueue(anyhow::anyhow!("queue")).is_transient());
}
#[tokio::test]
-async fn users_with_access_delegates_to_entity_access() {
- let users = service()
- .users_with_access("doc_1", EntityType::Document)
+async fn entity_access_internal_error_is_transient() {
+ let access = MockAccessService {
+ users: Vec::new(),
+ failure: Some(AccessFailure::Internal),
+ calls: Arc::default(),
+ };
+ let repository = MockRepository::new(Vec::new(), Vec::new());
+ let enqueuer = MockEnqueuer::default();
+ let service = service(access, repository, enqueuer);
+ let event = document_event_cases().remove(0).event;
+
+ let error = event
+ .ingest(&service)
.await
- .expect("mock resolves users");
- assert_eq!(users, vec![user_id("macro|[email protected]")]);
+ .expect_err("access adapter fails");
+
+ assert!(matches!(
+ error,
+ WebhookEventIngestionError::EntityAccess(AccessError::Internal)
+ ));
+ assert!(error.is_transient());
}
diff --git a/rust/cloud-storage/webhook/src/domain/mod.rs b/rust/cloud-storage/webhook/src/domain/mod.rs
index ea2b4e5c8f..79230450ad 100644
--- a/rust/cloud-storage/webhook/src/domain/mod.rs
+++ b/rust/cloud-storage/webhook/src/domain/mod.rs
@@ -1,5 +1,8 @@
//! Domain layer for webhooks.
+#[cfg(feature = "ports")]
+/// Webhook event delivery state machine and retry policy.
+pub mod delivery;
#[cfg(feature = "ingestion")]
/// Webhook event ingestion service.
pub mod ingestion;
diff --git a/rust/cloud-storage/webhook/src/domain/models.rs b/rust/cloud-storage/webhook/src/domain/models.rs
index e2edd489fd..f0ddf469bb 100644
--- a/rust/cloud-storage/webhook/src/domain/models.rs
+++ b/rust/cloud-storage/webhook/src/domain/models.rs
@@ -2,7 +2,7 @@
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
-use std::collections::BTreeMap;
+use std::{collections::BTreeMap, time::Duration};
/// Webhook id. Webhook ids are stored with a `wh_` prefix.
pub type WebhookId = String;
@@ -10,6 +10,261 @@ pub type WebhookId = String;
/// Custom headers supplied for webhook delivery.
pub type WebhookHeaders = BTreeMap<String, String>;
+/// Version of the webhook event queue message contract emitted by this crate.
+#[cfg(feature = "ports")]
+pub const WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION: u8 = 1;
+
+/// Event data normalized from a broker envelope for webhook matching and delivery.
+///
+/// `broker_envelope` retains the complete payload received from the broker and is
+/// the only value sent as the webhook HTTP request body.
+#[cfg(feature = "ports")]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct NormalizedWebhookEvent {
+ /// Broker event identifier, represented as a string for transport neutrality.
+ pub event_id: String,
+ /// Version of the broker event payload schema.
+ pub schema_version: u8,
+ /// Exact event name used for webhook filter matching.
+ pub event_name: String,
+ /// Normalized entity type used for access resolution and delivery records.
+ pub entity_type: String,
+ /// Entity identifier used for access resolution and webhook filter matching.
+ pub entity_id: String,
+ /// Key that preserves the event ordering observed during ingestion.
+ pub ordering_key: String,
+ /// Time at which the broker event was ingested for webhook processing.
+ pub occurred_at: DateTime<Utc>,
+ /// Complete broker envelope forwarded as the webhook HTTP request body.
+ pub broker_envelope: serde_json::Value,
+}
+
+/// Versioned queue payload for delivering one event to one webhook.
+///
+/// Endpoint URLs, custom headers, and signing secrets are deliberately absent;
+/// workers must resolve the webhook's current configuration before delivery.
+#[cfg(feature = "ports")]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+#[serde(deny_unknown_fields)]
+pub struct WebhookEventQueueMessage {
+ /// Queue contract version.
+ pub version: u8,
+ /// Webhook that should receive the event.
+ pub webhook_id: WebhookId,
+ /// Normalized broker event to deliver.
+ pub event: NormalizedWebhookEvent,
+}
+
+#[cfg(feature = "ports")]
+impl WebhookEventQueueMessage {
+ /// Create a queue message using the current contract version.
+ pub fn new(webhook_id: WebhookId, event: NormalizedWebhookEvent) -> Self {
+ Self {
+ version: WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION,
+ webhook_id,
+ event,
+ }
+ }
+
+ /// Return whether this message uses a contract version supported by this crate.
+ pub fn has_supported_version(&self) -> bool {
+ self.version == WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION
+ }
+}
+
+/// Raw queue message retained at the queue adapter boundary.
+///
+/// The optional body and receipt handle are kept separate so a worker can
+/// acknowledge malformed or missing payloads whenever the queue supplied a
+/// receipt handle.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct RawWebhookEventQueueMessage {
+ /// Queue-assigned message identifier, when supplied by the queue.
+ pub message_id: Option<String>,
+ /// Unparsed queue message body.
+ pub body: Option<String>,
+ /// Handle used to acknowledge or delay this receipt.
+ pub receipt_handle: Option<String>,
+}
+
+/// Identifier of a persisted webhook delivery.
+pub type WebhookDeliveryId = String;
+
+/// Identifier of a persisted webhook delivery attempt.
+pub type WebhookDeliveryAttemptId = String;
+
+/// Persisted lifecycle state of a webhook delivery.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum WebhookDeliveryStatus {
+ /// Delivery is persisted and ready for its first attempt.
+ Queued,
+ /// An HTTP attempt has started and has not completed.
+ InProgress,
+ /// A retryable attempt failed and another attempt is scheduled.
+ RetryScheduled,
+ /// The endpoint accepted the event.
+ Delivered,
+ /// Delivery ended after a non-retryable failure.
+ PermanentlyFailed,
+ /// Delivery exhausted the configured HTTP attempt limit.
+ Exhausted,
+ /// Delivery was canceled because the webhook is no longer eligible.
+ Canceled,
+}
+
+impl WebhookDeliveryStatus {
+ /// Storage representation.
+ pub const fn as_str(self) -> &'static str {
+ match self {
+ Self::Queued => "queued",
+ Self::InProgress => "in_progress",
+ Self::RetryScheduled => "retry_scheduled",
+ Self::Delivered => "delivered",
+ Self::PermanentlyFailed => "permanently_failed",
+ Self::Exhausted => "exhausted",
+ Self::Canceled => "canceled",
+ }
+ }
+
+ /// Return whether this status prevents any further HTTP attempts.
+ pub const fn is_terminal(self) -> bool {
+ matches!(
+ self,
+ Self::Delivered | Self::PermanentlyFailed | Self::Exhausted | Self::Canceled
+ )
+ }
+}
+
+impl std::str::FromStr for WebhookDeliveryStatus {
+ type Err = String;
+
+ fn from_str(value: &str) -> Result<Self, Self::Err> {
+ match value {
+ "queued" => Ok(Self::Queued),
+ "in_progress" => Ok(Self::InProgress),
+ "retry_scheduled" => Ok(Self::RetryScheduled),
+ "delivered" => Ok(Self::Delivered),
+ "permanently_failed" => Ok(Self::PermanentlyFailed),
+ "exhausted" => Ok(Self::Exhausted),
+ "canceled" => Ok(Self::Canceled),
+ other => Err(format!("unknown webhook delivery status: {other}")),
+ }
+ }
+}
+
+/// Persisted lifecycle state of one webhook delivery attempt.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum WebhookDeliveryAttemptStatus {
+ /// The HTTP request has started and has not completed.
+ InProgress,
+ /// The endpoint accepted the request.
+ Succeeded,
+ /// The request failed with a retryable outcome.
+ RetryableFailure,
+ /// The request failed with a permanent outcome.
+ PermanentFailure,
+ /// The request failed and exhausted the delivery's attempt limit.
+ Exhausted,
+ /// Processing stopped before the attempt outcome could be recorded.
+ Interrupted,
+}
+
+impl WebhookDeliveryAttemptStatus {
+ /// Storage representation.
+ pub const fn as_str(self) -> &'static str {
+ match self {
+ Self::InProgress => "in_progress",
+ Self::Succeeded => "succeeded",
+ Self::RetryableFailure => "retryable_failure",
+ Self::PermanentFailure => "permanent_failure",
+ Self::Exhausted => "exhausted",
+ Self::Interrupted => "interrupted",
+ }
+ }
+}
+
+impl std::str::FromStr for WebhookDeliveryAttemptStatus {
+ type Err = String;
+
+ fn from_str(value: &str) -> Result<Self, Self::Err> {
+ match value {
+ "in_progress" => Ok(Self::InProgress),
+ "succeeded" => Ok(Self::Succeeded),
+ "retryable_failure" => Ok(Self::RetryableFailure),
+ "permanent_failure" => Ok(Self::PermanentFailure),
+ "exhausted" => Ok(Self::Exhausted),
+ "interrupted" => Ok(Self::Interrupted),
+ other => Err(format!("unknown webhook delivery attempt status: {other}")),
+ }
+ }
+}
+
+/// Transport-neutral metadata captured from one webhook HTTP request.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WebhookHttpOutcomeDetails {
+ /// Time spent making the HTTP request.
+ pub duration: Duration,
+ /// HTTP response status, when the endpoint returned a response.
+ pub response_status: Option<u16>,
+ /// Response header names with values redacted by the delivery client.
+ pub response_headers_redacted: Option<WebhookHeaders>,
+ /// Lossy response-body preview, capped by the delivery client.
+ pub response_body_preview: Option<String>,
+ /// Sanitized failure category, when the request failed.
+ pub error_kind: Option<String>,
+ /// Sanitized failure message, when the request failed.
+ pub error_message: Option<String>,
+}
+
+/// Classification and metadata returned by a webhook HTTP delivery client.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum WebhookHttpOutcome {
+ /// The endpoint returned a successful response.
+ Success(WebhookHttpOutcomeDetails),
+ /// The request may succeed if attempted again.
+ RetryableFailure(WebhookHttpOutcomeDetails),
+ /// The request must not be attempted again.
+ PermanentFailure(WebhookHttpOutcomeDetails),
+}
+
+/// Current persisted delivery and webhook configuration loaded idempotently.
+#[derive(Debug, Clone)]
+pub struct PreparedWebhookDelivery {
+ /// Persisted delivery identifier.
+ pub delivery_id: WebhookDeliveryId,
+ /// Current webhook configuration, including outbound-only signing material.
+ pub webhook: Webhook,
+ /// Current persisted delivery status.
+ pub status: WebhookDeliveryStatus,
+ /// Number of HTTP attempts that have already started.
+ pub attempt_count: u32,
+ /// Earliest time at which another attempt may begin.
+ pub next_attempt_at: Option<DateTime<Utc>>,
+}
+
+/// Numbered webhook delivery attempt started by the repository.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WebhookDeliveryAttempt {
+ /// Persisted attempt identifier.
+ pub attempt_id: WebhookDeliveryAttemptId,
+ /// Delivery to which this attempt belongs.
+ pub delivery_id: WebhookDeliveryId,
+ /// One-based attempt number.
+ pub attempt_number: u32,
+}
+
+/// Action the inbound worker should apply to a processed queue receipt.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum WebhookWorkerDisposition {
+ /// Delete the queue receipt.
+ Acknowledge,
+ /// Make the queue receipt available again after the supplied delay.
+ RetryAfter(Duration),
+}
+
/// Event and optional entity-id constraints used to match webhook deliveries.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "inbound", derive(utoipa::ToSchema))]
@@ -73,6 +328,28 @@ pub enum WebhookScope {
Team,
}
+/// Endpoint restrictions applied to webhook URLs.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub enum WebhookEndpointSchemePolicy {
+ /// Require HTTPS endpoints with public addresses.
+ #[default]
+ HttpsOnly,
+ /// Allow HTTP and non-public addresses for local testing.
+ HttpAndHttps,
+}
+
+impl WebhookEndpointSchemePolicy {
+ /// Return whether the URL scheme is permitted by this policy.
+ pub fn allows(self, scheme: &str) -> bool {
+ scheme == "https" || (self == Self::HttpAndHttps && scheme == "http")
+ }
+
+ /// Return whether localhost, loopback, private, and link-local addresses are permitted.
+ pub fn allows_local_addresses(self) -> bool {
+ self == Self::HttpAndHttps
+ }
+}
+
/// Request to create a webhook.
#[derive(Debug, Clone, Deserialize)]
#[cfg_attr(feature = "inbound", derive(utoipa::ToSchema))]
@@ -81,7 +358,7 @@ pub struct CreateWebhookRequest {
pub scope: WebhookScope,
/// Display name.
pub name: String,
- /// HTTPS endpoint URL.
+ /// Endpoint URL. HTTPS is required outside local environments.
pub endpoint_url: String,
/// Optional custom delivery headers.
pub headers: Option<WebhookHeaders>,
@@ -95,7 +372,7 @@ pub struct CreateWebhookRequest {
pub struct PatchWebhookRequest {
/// Display name.
pub name: Option<String>,
- /// HTTPS endpoint URL.
+ /// Endpoint URL. HTTPS is required outside local environments.
pub endpoint_url: Option<String>,
/// Optional custom delivery headers. When present, replaces existing headers.
pub headers: Option<WebhookHeaders>,
@@ -115,7 +392,7 @@ pub struct Webhook {
pub workspace_id: String,
/// Display name.
pub name: String,
- /// HTTPS endpoint URL.
+ /// Endpoint URL. HTTPS is required outside local environments.
pub endpoint_url: String,
/// Signing secret used by outbound adapters. This is never serialized by APIs.
#[serde(default, skip_serializing)]
diff --git a/rust/cloud-storage/webhook/src/domain/models_test.rs b/rust/cloud-storage/webhook/src/domain/models_test.rs
index 5e1154d331..126f475b38 100644
--- a/rust/cloud-storage/webhook/src/domain/models_test.rs
+++ b/rust/cloud-storage/webhook/src/domain/models_test.rs
@@ -1,4 +1,10 @@
+#[cfg(feature = "ports")]
+use super::models::{
+ NormalizedWebhookEvent, WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION, WebhookEventQueueMessage,
+};
use super::models::{WebhookFilter, WebhookFilters};
+#[cfg(feature = "ports")]
+use chrono::{DateTime, Utc};
use serde_json::json;
#[test]
@@ -74,3 +80,78 @@ fn rejects_non_string_array_entries() {
assert!(result.is_err());
}
}
+
+#[cfg(feature = "ports")]
+#[test]
+fn queue_message_json_round_trip_retains_normalized_event() {
+ let broker_envelope = json!({
+ "event_id": "019bbb3e-8d80-7000-8000-000000000001",
+ "schema_version": 2,
+ "event_type": "document.created",
+ "metadata": {
+ "document_id": "doc_123",
+ "name": "Quarterly plan",
+ },
+ });
+ let occurred_at = DateTime::parse_from_rfc3339("2026-07-10T15:30:00Z")
+ .unwrap()
+ .with_timezone(&Utc);
+ let message = WebhookEventQueueMessage::new(
+ "wh_123".to_string(),
+ NormalizedWebhookEvent {
+ event_id: "019bbb3e-8d80-7000-8000-000000000001".to_string(),
+ schema_version: 2,
+ event_name: "document.created".to_string(),
+ entity_type: "document".to_string(),
+ entity_id: "doc_123".to_string(),
+ ordering_key: "doc_123".to_string(),
+ occurred_at,
+ broker_envelope: broker_envelope.clone(),
+ },
+ );
+
+ let serialized = serde_json::to_value(&message).unwrap();
+ assert_eq!(
+ serialized,
+ json!({
+ "version": WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION,
+ "webhook_id": "wh_123",
+ "event": {
+ "event_id": "019bbb3e-8d80-7000-8000-000000000001",
+ "schema_version": 2,
+ "event_name": "document.created",
+ "entity_type": "document",
+ "entity_id": "doc_123",
+ "ordering_key": "doc_123",
+ "occurred_at": "2026-07-10T15:30:00Z",
+ "broker_envelope": broker_envelope,
+ },
+ })
+ );
+
+ let round_trip: WebhookEventQueueMessage = serde_json::from_value(serialized).unwrap();
+ assert_eq!(round_trip, message);
+ assert!(round_trip.has_supported_version());
+}
+
+#[cfg(feature = "ports")]
+#[test]
+fn queue_message_rejects_delivery_configuration() {
+ let result = serde_json::from_value::<WebhookEventQueueMessage>(json!({
+ "version": WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION,
+ "webhook_id": "wh_123",
+ "endpoint_url": "https://example.com/webhook",
+ "event": {
+ "event_id": "evt_123",
+ "schema_version": 1,
+ "event_name": "document.created",
+ "entity_type": "document",
+ "entity_id": "doc_123",
+ "ordering_key": "doc_123",
+ "occurred_at": "2026-07-10T15:30:00Z",
+ "broker_envelope": {},
+ },
+ }));
+
+ assert!(result.is_err());
+}
diff --git a/rust/cloud-storage/webhook/src/domain/ports.rs b/rust/cloud-storage/webhook/src/domain/ports.rs
index 09507ff053..95bb57b3db 100644
--- a/rust/cloud-storage/webhook/src/domain/ports.rs
+++ b/rust/cloud-storage/webhook/src/domain/ports.rs
@@ -1,11 +1,14 @@
//! Webhook domain ports.
use super::models::{
- CreateWebhookRequest, PatchWebhookRequest, ValidateWebhookResponse, Webhook, WebhookId,
- WebhookValidationResult,
+ CreateWebhookRequest, NormalizedWebhookEvent, PatchWebhookRequest, PreparedWebhookDelivery,
+ RawWebhookEventQueueMessage, ValidateWebhookResponse, Webhook, WebhookDeliveryAttempt,
+ WebhookEventQueueMessage, WebhookHttpOutcome, WebhookHttpOutcomeDetails, WebhookId,
+ WebhookValidationResult, WebhookWorkerDisposition,
};
+use chrono::{DateTime, Utc};
use macro_user_id::user_id::MacroUserIdStr;
-use std::future::Future;
+use std::{future::Future, time::Duration};
/// Webhook repository.
pub trait WebhookRepo: Clone + Send + Sync + 'static {
@@ -66,6 +69,138 @@ pub trait WebhookRepo: Clone + Send + Sync + 'static {
) -> impl Future<Output = Result<Option<String>, Self::Err>> + Send;
}
+/// Resolver that expands people into personal and team webhook workspace ids.
+pub trait WebhookWorkspaceResolver: Clone + Send + Sync + 'static {
+ /// Resolver error.
+ type Err: Into<anyhow::Error> + Send;
+
+ /// Resolve a deterministic, deduplicated set of personal and team workspace ids.
+ fn resolve_workspace_ids(
+ &self,
+ people: Vec<MacroUserIdStr<'static>>,
+ ) -> impl Future<Output = Result<Vec<String>, Self::Err>> + Send;
+}
+
+/// Port for enqueueing one normalized event for one matched webhook.
+pub trait WebhookEventEnqueuer: Clone + Send + Sync + 'static {
+ /// Enqueuer error.
+ type Err: Into<anyhow::Error> + Send;
+
+ /// Enqueue one webhook event delivery.
+ fn enqueue(
+ &self,
+ message: WebhookEventQueueMessage,
+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;
+}
+
+/// Queue operations used by the inbound webhook event worker.
+pub trait WebhookEventQueue: Clone + Send + Sync + 'static {
+ /// Queue error.
+ type Err: Into<anyhow::Error> + Send;
+
+ /// Receive a batch of unparsed queue messages.
+ fn receive_messages(
+ &self,
+ ) -> impl Future<Output = Result<Vec<RawWebhookEventQueueMessage>, Self::Err>> + Send;
+
+ /// Delete a queue receipt after processing or poison-message handling.
+ fn delete_message(
+ &self,
+ receipt_handle: &str,
+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;
+
+ /// Delay redelivery of a queue receipt by changing its visibility period.
+ fn delay_message(
+ &self,
+ receipt_handle: &str,
+ delay: Duration,
+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;
+}
+
+/// Repository for idempotent webhook deliveries and atomic attempt transitions.
+pub trait WebhookDeliveryRepository: Clone + Send + Sync + 'static {
+ /// Repository error.
+ type Err: Into<anyhow::Error> + Send;
+
+ /// Create a delivery if needed and load its current state and webhook configuration.
+ ///
+ /// Implementations must enforce idempotency for the message's webhook and event ids.
+ /// `None` indicates that the webhook does not exist and no delivery could be prepared.
+ fn prepare_delivery(
+ &self,
+ message: &WebhookEventQueueMessage,
+ ) -> impl Future<Output = Result<Option<PreparedWebhookDelivery>, Self::Err>> + Send;
+
+ /// Mark a prepared, non-terminal delivery as canceled without creating an attempt.
+ fn cancel_delivery(
+ &self,
+ delivery_id: &str,
+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;
+
+ /// Lock a delivery row and begin its next one-based attempt.
+ ///
+ /// Implementations must re-check status and `next_attempt_at` while holding the
+ /// row lock. `None` means the delivery is terminal or is not ready for an attempt.
+ fn begin_attempt(
+ &self,
+ delivery_id: &str,
+ ) -> impl Future<Output = Result<Option<WebhookDeliveryAttempt>, Self::Err>> + Send;
+
+ /// Atomically record a successful attempt and delivered webhook state.
+ fn record_success(
+ &self,
+ attempt: &WebhookDeliveryAttempt,
+ details: &WebhookHttpOutcomeDetails,
+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;
+
+ /// Atomically record a retryable failure and schedule the next attempt.
+ fn record_retryable_failure(
+ &self,
+ attempt: &WebhookDeliveryAttempt,
+ details: &WebhookHttpOutcomeDetails,
+ next_attempt_at: DateTime<Utc>,
+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;
+
+ /// Atomically record an attempt and delivery as permanently failed.
+ fn record_permanent_failure(
+ &self,
+ attempt: &WebhookDeliveryAttempt,
+ details: &WebhookHttpOutcomeDetails,
+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;
+
+ /// Atomically record a failed attempt and delivery as exhausted.
+ fn record_exhaustion(
+ &self,
+ attempt: &WebhookDeliveryAttempt,
+ details: &WebhookHttpOutcomeDetails,
+ ) -> impl Future<Output = Result<(), Self::Err>> + Send;
+}
+
+/// Client that sends one signed webhook request using current webhook configuration.
+pub trait WebhookDeliveryClient: Clone + Send + Sync + 'static {
+ /// Client error for failures that cannot be represented as an HTTP outcome.
+ type Err: Into<anyhow::Error> + Send;
+
+ /// Deliver the complete broker envelope to a webhook endpoint.
+ fn deliver(
+ &self,
+ webhook: &Webhook,
+ event: &NormalizedWebhookEvent,
+ ) -> impl Future<Output = Result<WebhookHttpOutcome, Self::Err>> + Send;
+}
+
+/// Domain service that prepares, attempts, and records one queued webhook event.
+pub trait WebhookEventDeliveryService: Clone + Send + Sync + 'static {
+ /// Delivery service error.
+ type Err: Into<anyhow::Error> + Send;
+
+ /// Process one valid queue payload and choose how its receipt should be handled.
+ fn deliver_event(
+ &self,
+ message: WebhookEventQueueMessage,
+ ) -> impl Future<Output = Result<WebhookWorkerDisposition, Self::Err>> + Send;
+}
+
/// Client used to send a signed webhook validation delivery.
pub trait WebhookValidationClient: Clone + Send + Sync + 'static {
/// Client error.
diff --git a/rust/cloud-storage/webhook/src/domain/service.rs b/rust/cloud-storage/webhook/src/domain/service.rs
index b90f9ccfd4..74a6fabfe8 100644
--- a/rust/cloud-storage/webhook/src/domain/service.rs
+++ b/rust/cloud-storage/webhook/src/domain/service.rs
@@ -3,7 +3,8 @@
use super::{
models::{
CreateWebhookRequest, PatchWebhookRequest, ValidateWebhookResponse, Webhook,
- WebhookFilters, WebhookId, WebhookScope, WebhookValidationResult,
+ WebhookEndpointSchemePolicy, WebhookFilters, WebhookId, WebhookScope,
+ WebhookValidationResult,
},
ports::{WebhookError, WebhookRepo, WebhookService, WebhookValidationClient},
};
@@ -24,14 +25,29 @@ const VALIDATION_EVENT_NAME: &str = "webhook.validation.test";
pub struct WebhookServiceImpl<R, V> {
repo: R,
validation_client: V,
+ endpoint_scheme_policy: WebhookEndpointSchemePolicy,
}
impl<R, V> WebhookServiceImpl<R, V> {
- /// Create a webhook service.
+ /// Create a webhook service that requires public HTTPS endpoints.
pub fn new(repo: R, validation_client: V) -> Self {
+ Self::new_with_endpoint_scheme_policy(
+ repo,
+ validation_client,
+ WebhookEndpointSchemePolicy::HttpsOnly,
+ )
+ }
+
+ /// Create a webhook service with an explicit endpoint policy.
+ pub fn new_with_endpoint_scheme_policy(
+ repo: R,
+ validation_client: V,
+ endpoint_scheme_policy: WebhookEndpointSchemePolicy,
+ ) -> Self {
Self {
repo,
validation_client,
+ endpoint_scheme_policy,
}
}
}
@@ -46,11 +62,14 @@ fn validate_name(name: &str) -> Result<(), WebhookError> {
Ok(())
}
-fn validate_endpoint_url(endpoint_url: &str) -> Result<(), WebhookError> {
+fn validate_endpoint_url(
+ endpoint_url: &str,
+ endpoint_scheme_policy: WebhookEndpointSchemePolicy,
+) -> Result<(), WebhookError> {
let url = Url::parse(endpoint_url)
.map_err(|_| WebhookError::BadRequest("endpoint_url must be a valid URL".to_string()))?;
- if url.scheme() != "https" {
+ if !endpoint_scheme_policy.allows(url.scheme()) {
return Err(WebhookError::BadRequest(
"endpoint_url must use https".to_string(),
));
@@ -62,7 +81,7 @@ fn validate_endpoint_url(endpoint_url: &str) -> Result<(), WebhookError> {
));
};
- if is_blocked_endpoint_host(host) {
+ if !endpoint_scheme_policy.allows_local_addresses() && is_blocked_endpoint_host(host) {
return Err(WebhookError::BadRequest(
"endpoint_url host is not allowed".to_string(),
));
@@ -171,18 +190,24 @@ fn validate_filters(filters: &WebhookFilters) -> Result<(), WebhookError> {
Ok(())
}
-fn validate_create_request(request: &CreateWebhookRequest) -> Result<(), WebhookError> {
+fn validate_create_request(
+ request: &CreateWebhookRequest,
+ endpoint_scheme_policy: WebhookEndpointSchemePolicy,
+) -> Result<(), WebhookError> {
validate_name(&request.name)?;
- validate_endpoint_url(&request.endpoint_url)?;
+ validate_endpoint_url(&request.endpoint_url, endpoint_scheme_policy)?;
validate_filters(&request.filters)
}
-fn validate_patch_request(request: &PatchWebhookRequest) -> Result<(), WebhookError> {
+fn validate_patch_request(
+ request: &PatchWebhookRequest,
+ endpoint_scheme_policy: WebhookEndpointSchemePolicy,
+) -> Result<(), WebhookError> {
if let Some(name) = &request.name {
validate_name(name)?;
}
if let Some(endpoint_url) = &request.endpoint_url {
- validate_endpoint_url(endpoint_url)?;
+ validate_endpoint_url(endpoint_url, endpoint_scheme_policy)?;
}
if let Some(filters) = &request.filters {
validate_filters(filters)?;
@@ -291,7 +316,7 @@ where
caller: MacroUserIdStr<'static>,
request: CreateWebhookRequest,
) -> Result<Webhook, WebhookError> {
- validate_create_request(&request)?;
+ validate_create_request(&request, self.endpoint_scheme_policy)?;
let workspace_id = self
.workspace_id_for_scope(caller.clone(), request.scope)
@@ -318,7 +343,7 @@ where
let webhook = self
.load_authorized_webhook(caller, webhook_id.clone())
.await?;
- validate_patch_request(&request)?;
+ validate_patch_request(&request, self.endpoint_scheme_policy)?;
let reset_validity = request.endpoint_url.is_some() || request.headers.is_some();
let patched = self
diff --git a/rust/cloud-storage/webhook/src/domain/service_test.rs b/rust/cloud-storage/webhook/src/domain/service_test.rs
index 52572e7bb4..617c1f22fc 100644
--- a/rust/cloud-storage/webhook/src/domain/service_test.rs
+++ b/rust/cloud-storage/webhook/src/domain/service_test.rs
@@ -1,7 +1,8 @@
use super::{
models::{
- CreateWebhookRequest, PatchWebhookRequest, ValidateWebhookResponse, Webhook, WebhookFilter,
- WebhookFilters, WebhookScope, WebhookStatus, WebhookValidationResult,
+ CreateWebhookRequest, PatchWebhookRequest, ValidateWebhookResponse, Webhook,
+ WebhookEndpointSchemePolicy, WebhookFilter, WebhookFilters, WebhookScope, WebhookStatus,
+ WebhookValidationResult,
},
ports::{WebhookError, WebhookRepo, WebhookService, WebhookValidationClient},
service::WebhookServiceImpl,
@@ -352,6 +353,33 @@ async fn invalid_http_endpoint_is_rejected() {
assert_bad_request(service.create_webhook(caller(), request).await);
}
+#[tokio::test]
+async fn local_addresses_are_allowed_when_policy_permits_them() {
+ let service = WebhookServiceImpl::new_with_endpoint_scheme_policy(
+ FakeRepo::default(),
+ FakeValidationClient::default(),
+ WebhookEndpointSchemePolicy::HttpAndHttps,
+ );
+
+ for endpoint_url in [
+ "http://localhost/webhook",
+ "http://127.0.0.1/webhook",
+ "http://10.1.2.3/webhook",
+ "http://[::1]/webhook",
+ "http://[fe80::1]/webhook",
+ ] {
+ let mut request = create_request();
+ request.endpoint_url = endpoint_url.to_string();
+
+ let webhook = service
+ .create_webhook(caller(), request)
+ .await
+ .expect("local address should be allowed");
+
+ assert_eq!(webhook.endpoint_url, endpoint_url);
+ }
+}
+
#[tokio::test]
async fn private_and_link_local_endpoints_are_rejected() {
let repo = FakeRepo::default();
diff --git a/rust/cloud-storage/webhook/src/inbound/kafka_consumer.rs b/rust/cloud-storage/webhook/src/inbound/kafka_consumer.rs
index 58491055d5..c60eb65370 100644
--- a/rust/cloud-storage/webhook/src/inbound/kafka_consumer.rs
+++ b/rust/cloud-storage/webhook/src/inbound/kafka_consumer.rs
@@ -287,7 +287,6 @@ where
match WebhookConsumerEvent::decode(message.topic(), payload) {
Ok(event) => {
tracing::trace!(
- ?event,
partition = message.partition(),
offset = message.offset(),
"decoded broker event"
diff --git a/rust/cloud-storage/webhook/src/inbound/kafka_consumer/test.rs b/rust/cloud-storage/webhook/src/inbound/kafka_consumer/test.rs
index e8d129f76b..acc0274c64 100644
--- a/rust/cloud-storage/webhook/src/inbound/kafka_consumer/test.rs
+++ b/rust/cloud-storage/webhook/src/inbound/kafka_consumer/test.rs
@@ -72,11 +72,9 @@ struct FlakyIngestionService {
impl FlakyIngestionService {
fn failure(&self) -> WebhookEventIngestionError {
if self.transient {
- WebhookEventIngestionError::EntityAccess(AccessError::DatabaseError(
- sqlx::Error::PoolTimedOut,
- ))
+ WebhookEventIngestionError::Enqueue(anyhow::anyhow!("queue unavailable"))
} else {
- WebhookEventIngestionError::Internal(anyhow::anyhow!("permanent failure"))
+ WebhookEventIngestionError::EntityAccess(AccessError::Unauthorized)
}
}
}
diff --git a/rust/cloud-storage/webhook/src/inbound/mod.rs b/rust/cloud-storage/webhook/src/inbound/mod.rs
index 2b84ac5b7d..6efff1c630 100644
--- a/rust/cloud-storage/webhook/src/inbound/mod.rs
+++ b/rust/cloud-storage/webhook/src/inbound/mod.rs
@@ -1,4 +1,4 @@
-//! Inbound adapters: HTTP handlers and the Kafka event consumer.
+//! Inbound adapters: HTTP handlers, the Kafka event consumer, and queue worker.
#[cfg(feature = "inbound")]
pub mod axum_router;
@@ -8,3 +8,6 @@ mod axum_router_test;
#[cfg(feature = "consumer")]
pub mod kafka_consumer;
+
+#[cfg(feature = "worker")]
+pub mod worker;
diff --git a/rust/cloud-storage/webhook/src/inbound/worker.rs b/rust/cloud-storage/webhook/src/inbound/worker.rs
new file mode 100644
index 0000000000..5f1bd94291
--- /dev/null
+++ b/rust/cloud-storage/webhook/src/inbound/worker.rs
@@ -0,0 +1,152 @@
+//! Inbound worker for queued webhook events.
+//!
+//! The worker owns queue polling and receipt handling. Delivery state, HTTP
+//! attempts, and retry policy remain in the domain delivery service.
+
+#[cfg(test)]
+mod test;
+
+use std::time::Duration;
+
+use crate::domain::{
+ models::{RawWebhookEventQueueMessage, WebhookEventQueueMessage, WebhookWorkerDisposition},
+ ports::{WebhookEventDeliveryService, WebhookEventQueue},
+};
+
+const POLL_RETRY_DELAY: Duration = Duration::from_secs(1);
+
+/// Continuously receives and processes queued webhook events.
+///
+/// Messages in each received batch are processed sequentially so events from
+/// the same FIFO message group cannot be reordered within a batch.
+pub struct WebhookEventWorker<Q, S> {
+ queue: Q,
+ service: S,
+}
+
+impl<Q, S> WebhookEventWorker<Q, S>
+where
+ Q: WebhookEventQueue,
+ S: WebhookEventDeliveryService,
+{
+ /// Create a worker using the queue transport and delivery domain service.
+ pub fn new(queue: Q, service: S) -> Self {
+ Self { queue, service }
+ }
+
+ /// Run the webhook event worker until the task is canceled.
+ pub async fn run(&self) -> ! {
+ loop {
+ self.poll_and_process_batch().await;
+ }
+ }
+
+ async fn poll_and_process_batch(&self) {
+ let messages = match self.queue.receive_messages().await {
+ Ok(messages) => messages,
+ Err(error) => {
+ let error: anyhow::Error = error.into();
+ tracing::error!(error = ?error, "failed to receive webhook event messages");
+ tokio::time::sleep(POLL_RETRY_DELAY).await;
+ return;
+ }
+ };
+
+ if messages.is_empty() {
+ tokio::time::sleep(POLL_RETRY_DELAY).await;
+ return;
+ }
+
+ tracing::debug!(count = messages.len(), "received webhook event messages");
+ for message in messages {
+ self.process_message(message).await;
+ }
+ }
+
+ #[tracing::instrument(
+ skip(self, message),
+ fields(message_id = message.message_id.as_deref().unwrap_or("unknown"))
+ )]
+ async fn process_message(&self, message: RawWebhookEventQueueMessage) {
+ let Some(body) = message.body.as_deref() else {
+ tracing::warn!("acknowledging webhook event message without a body");
+ self.acknowledge_poison_message(message.receipt_handle.as_deref())
+ .await;
+ return;
+ };
+
+ let queue_message = match serde_json::from_str::<WebhookEventQueueMessage>(body) {
+ Ok(queue_message) => queue_message,
+ Err(error) => {
+ tracing::warn!(
+ error = ?error,
+ "acknowledging malformed webhook event message"
+ );
+ self.acknowledge_poison_message(message.receipt_handle.as_deref())
+ .await;
+ return;
+ }
+ };
+
+ if !queue_message.has_supported_version() {
+ tracing::warn!(
+ version = queue_message.version,
+ "acknowledging unsupported webhook event message version"
+ );
+ self.acknowledge_poison_message(message.receipt_handle.as_deref())
+ .await;
+ return;
+ }
+
+ let Some(receipt_handle) = message.receipt_handle.as_deref() else {
+ tracing::error!(
+ webhook_id = %queue_message.webhook_id,
+ event_id = %queue_message.event.event_id,
+ "cannot process webhook event message without a receipt handle"
+ );
+ return;
+ };
+
+ let disposition = match self.service.deliver_event(queue_message).await {
+ Ok(disposition) => disposition,
+ Err(error) => {
+ let error: anyhow::Error = error.into();
+ tracing::error!(error = ?error, "failed to process webhook event message");
+ return;
+ }
+ };
+
+ match disposition {
+ WebhookWorkerDisposition::Acknowledge => {
+ if let Err(error) = self.queue.delete_message(receipt_handle).await {
+ let error: anyhow::Error = error.into();
+ tracing::error!(error = ?error, "failed to acknowledge webhook event message");
+ }
+ }
+ WebhookWorkerDisposition::RetryAfter(delay) => {
+ if let Err(error) = self.queue.delay_message(receipt_handle, delay).await {
+ let error: anyhow::Error = error.into();
+ tracing::error!(
+ error = ?error,
+ delay_seconds = delay.as_secs(),
+ "failed to update webhook event message visibility"
+ );
+ }
+ }
+ }
+ }
+
+ async fn acknowledge_poison_message(&self, receipt_handle: Option<&str>) {
+ let Some(receipt_handle) = receipt_handle else {
+ tracing::error!(
+ "cannot acknowledge poison webhook event message without a receipt handle"
+ );
+ return;
+ };
+
+ if let Err(error) = self.queue.delete_message(receipt_handle).await {
+ let error: anyhow::Error = error.into();
+ tracing::error!(error = ?error, "failed to acknowledge poison webhook event message");
+ }
+ }
+}
diff --git a/rust/cloud-storage/webhook/src/inbound/worker/test.rs b/rust/cloud-storage/webhook/src/inbound/worker/test.rs
new file mode 100644
index 0000000000..3974164b6d
--- /dev/null
+++ b/rust/cloud-storage/webhook/src/inbound/worker/test.rs
@@ -0,0 +1,391 @@
+use super::*;
+use crate::domain::models::{NormalizedWebhookEvent, WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION};
+use chrono::{DateTime, Utc};
+use std::{
+ collections::{HashSet, VecDeque},
+ fmt,
+ sync::{Arc, Mutex},
+};
+use tokio::time::Instant;
+
+#[derive(Debug, Clone, Copy)]
+struct TestError(&'static str);
+
+impl fmt::Display for TestError {
+ fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+ formatter.write_str(self.0)
+ }
+}
+
+impl std::error::Error for TestError {}
+
+#[derive(Default)]
+struct QueueState {
+ receive_results: VecDeque<Result<Vec<RawWebhookEventQueueMessage>, TestError>>,
+ receive_count: usize,
+ delete_requests: Vec<String>,
+ delay_requests: Vec<(String, Duration)>,
+ failed_deletes: HashSet<String>,
+ failed_delays: HashSet<String>,
+ operations: Option<Arc<Mutex<Vec<String>>>>,
+}
+
+#[derive(Clone, Default)]
+struct TestQueue {
+ state: Arc<Mutex<QueueState>>,
+}
+
+impl TestQueue {
+ fn with_receive_result(result: Result<Vec<RawWebhookEventQueueMessage>, TestError>) -> Self {
+ let queue = Self::default();
+ queue
+ .state
+ .lock()
+ .expect("queue state lock")
+ .receive_results
+ .push_back(result);
+ queue
+ }
+
+ fn fail_delete(&self, receipt_handle: &str) {
+ self.state
+ .lock()
+ .expect("queue state lock")
+ .failed_deletes
+ .insert(receipt_handle.to_string());
+ }
+
+ fn fail_delay(&self, receipt_handle: &str) {
+ self.state
+ .lock()
+ .expect("queue state lock")
+ .failed_delays
+ .insert(receipt_handle.to_string());
+ }
+
+ fn set_operations(&self, operations: Arc<Mutex<Vec<String>>>) {
+ self.state.lock().expect("queue state lock").operations = Some(operations);
+ }
+
+ fn receive_count(&self) -> usize {
+ self.state.lock().expect("queue state lock").receive_count
+ }
+
+ fn delete_requests(&self) -> Vec<String> {
+ self.state
+ .lock()
+ .expect("queue state lock")
+ .delete_requests
+ .clone()
+ }
+
+ fn delay_requests(&self) -> Vec<(String, Duration)> {
+ self.state
+ .lock()
+ .expect("queue state lock")
+ .delay_requests
+ .clone()
+ }
+}
+
+impl WebhookEventQueue for TestQueue {
+ type Err = TestError;
+
+ async fn receive_messages(&self) -> Result<Vec<RawWebhookEventQueueMessage>, Self::Err> {
+ let mut state = self.state.lock().expect("queue state lock");
+ state.receive_count += 1;
+ state.receive_results.pop_front().unwrap_or(Ok(Vec::new()))
+ }
+
+ async fn delete_message(&self, receipt_handle: &str) -> Result<(), Self::Err> {
+ let mut state = self.state.lock().expect("queue state lock");
+ state.delete_requests.push(receipt_handle.to_string());
+ if let Some(operations) = &state.operations {
+ operations
+ .lock()
+ .expect("operations lock")
+ .push(format!("delete:{receipt_handle}"));
+ }
+ if state.failed_deletes.contains(receipt_handle) {
+ return Err(TestError("delete failed"));
+ }
+ Ok(())
+ }
+
+ async fn delay_message(&self, receipt_handle: &str, delay: Duration) -> Result<(), Self::Err> {
+ let mut state = self.state.lock().expect("queue state lock");
+ state
+ .delay_requests
+ .push((receipt_handle.to_string(), delay));
+ if state.failed_delays.contains(receipt_handle) {
+ return Err(TestError("visibility update failed"));
+ }
+ Ok(())
+ }
+}
+
+struct ServiceState {
+ outcomes: VecDeque<Result<WebhookWorkerDisposition, TestError>>,
+ messages: Vec<WebhookEventQueueMessage>,
+ operations: Option<Arc<Mutex<Vec<String>>>>,
+}
+
+#[derive(Clone)]
+struct TestDeliveryService {
+ state: Arc<Mutex<ServiceState>>,
+}
+
+impl TestDeliveryService {
+ fn new(
+ outcomes: impl IntoIterator<Item = Result<WebhookWorkerDisposition, TestError>>,
+ ) -> Self {
+ Self {
+ state: Arc::new(Mutex::new(ServiceState {
+ outcomes: outcomes.into_iter().collect(),
+ messages: Vec::new(),
+ operations: None,
+ })),
+ }
+ }
+
+ fn set_operations(&self, operations: Arc<Mutex<Vec<String>>>) {
+ self.state.lock().expect("service state lock").operations = Some(operations);
+ }
+
+ fn messages(&self) -> Vec<WebhookEventQueueMessage> {
+ self.state
+ .lock()
+ .expect("service state lock")
+ .messages
+ .clone()
+ }
+}
+
+impl WebhookEventDeliveryService for TestDeliveryService {
+ type Err = TestError;
+
+ async fn deliver_event(
+ &self,
+ message: WebhookEventQueueMessage,
+ ) -> Result<WebhookWorkerDisposition, Self::Err> {
+ let mut state = self.state.lock().expect("service state lock");
+ if let Some(operations) = &state.operations {
+ operations
+ .lock()
+ .expect("operations lock")
+ .push(format!("service:{}", message.event.event_id));
+ }
+ state.messages.push(message);
+ state
+ .outcomes
+ .pop_front()
+ .expect("a delivery outcome for every valid test message")
+ }
+}
+
+fn queue_message(event_id: &str) -> WebhookEventQueueMessage {
+ WebhookEventQueueMessage::new(
+ "wh_test".to_string(),
+ NormalizedWebhookEvent {
+ event_id: event_id.to_string(),
+ schema_version: 1,
+ event_name: "document.updated".to_string(),
+ entity_type: "document".to_string(),
+ entity_id: "doc_test".to_string(),
+ ordering_key: "doc_test".to_string(),
+ occurred_at: "2026-07-10T00:00:00Z"
+ .parse::<DateTime<Utc>>()
+ .expect("valid timestamp"),
+ broker_envelope: serde_json::json!({"event_id": event_id}),
+ },
+ )
+}
+
+fn raw_message(
+ message_id: &str,
+ body: Option<String>,
+ receipt_handle: Option<&str>,
+) -> RawWebhookEventQueueMessage {
+ RawWebhookEventQueueMessage {
+ message_id: Some(message_id.to_string()),
+ body,
+ receipt_handle: receipt_handle.map(str::to_string),
+ }
+}
+
+fn valid_raw_message(event_id: &str, receipt_handle: &str) -> RawWebhookEventQueueMessage {
+ raw_message(
+ event_id,
+ Some(serde_json::to_string(&queue_message(event_id)).expect("serializable queue message")),
+ Some(receipt_handle),
+ )
+}
+
+#[tokio::test(start_paused = true)]
+async fn empty_poll_sleeps_before_polling_again() {
+ let queue = TestQueue::with_receive_result(Ok(Vec::new()));
+ let worker = WebhookEventWorker::new(queue.clone(), TestDeliveryService::new([]));
+ let started_at = Instant::now();
+
+ worker.poll_and_process_batch().await;
+
+ assert_eq!(Instant::now().duration_since(started_at), POLL_RETRY_DELAY);
+ assert_eq!(queue.receive_count(), 1);
+}
+
+#[tokio::test(start_paused = true)]
+async fn failed_poll_sleeps_before_polling_again() {
+ let queue = TestQueue::with_receive_result(Err(TestError("receive failed")));
+ let worker = WebhookEventWorker::new(queue.clone(), TestDeliveryService::new([]));
+ let started_at = Instant::now();
+
+ worker.poll_and_process_batch().await;
+
+ assert_eq!(Instant::now().duration_since(started_at), POLL_RETRY_DELAY);
+ assert_eq!(queue.receive_count(), 1);
+}
+
+#[tokio::test]
+async fn acknowledges_missing_malformed_and_unsupported_messages() {
+ let mut unsupported = queue_message("unsupported");
+ unsupported.version = WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION + 1;
+ let messages = vec![
+ raw_message("missing", None, Some("receipt-missing")),
+ raw_message(
+ "malformed",
+ Some("not valid json".to_string()),
+ Some("receipt-malformed"),
+ ),
+ raw_message(
+ "unsupported",
+ Some(serde_json::to_string(&unsupported).expect("serializable queue message")),
+ Some("receipt-unsupported"),
+ ),
+ ];
+ let queue = TestQueue::with_receive_result(Ok(messages));
+ let service = TestDeliveryService::new([]);
+ let worker = WebhookEventWorker::new(queue.clone(), service.clone());
+
+ worker.poll_and_process_batch().await;
+
+ assert_eq!(
+ queue.delete_requests(),
+ [
+ "receipt-missing",
+ "receipt-malformed",
+ "receipt-unsupported"
+ ]
+ );
+ assert!(service.messages().is_empty());
+}
+
+#[tokio::test]
+async fn deletes_valid_messages_after_acknowledge_disposition() {
+ let message = queue_message("event-1");
+ let queue = TestQueue::with_receive_result(Ok(vec![valid_raw_message(
+ &message.event.event_id,
+ "receipt-1",
+ )]));
+ let service = TestDeliveryService::new([Ok(WebhookWorkerDisposition::Acknowledge)]);
+ let worker = WebhookEventWorker::new(queue.clone(), service.clone());
+
+ worker.poll_and_process_batch().await;
+
+ assert_eq!(service.messages(), [message]);
+ assert_eq!(queue.delete_requests(), ["receipt-1"]);
+ assert!(queue.delay_requests().is_empty());
+}
+
+#[tokio::test]
+async fn changes_visibility_after_retry_disposition() {
+ let delay = Duration::from_secs(30);
+ let queue = TestQueue::with_receive_result(Ok(vec![valid_raw_message("event-1", "receipt-1")]));
+ let service = TestDeliveryService::new([Ok(WebhookWorkerDisposition::RetryAfter(delay))]);
+ let worker = WebhookEventWorker::new(queue.clone(), service);
+
+ worker.poll_and_process_batch().await;
+
+ assert_eq!(queue.delay_requests(), [("receipt-1".to_string(), delay)]);
+ assert!(queue.delete_requests().is_empty());
+}
+
+#[tokio::test]
+async fn failed_acknowledgment_leaves_the_message_for_redelivery() {
+ let queue = TestQueue::with_receive_result(Ok(vec![valid_raw_message("event-1", "receipt-1")]));
+ queue.fail_delete("receipt-1");
+ let service = TestDeliveryService::new([Ok(WebhookWorkerDisposition::Acknowledge)]);
+ let worker = WebhookEventWorker::new(queue.clone(), service);
+
+ worker.poll_and_process_batch().await;
+
+ assert_eq!(queue.delete_requests(), ["receipt-1"]);
+ assert!(queue.delay_requests().is_empty());
+}
+
+#[tokio::test]
+async fn failed_visibility_update_leaves_the_message_for_redelivery() {
+ let delay = Duration::from_secs(60);
+ let queue = TestQueue::with_receive_result(Ok(vec![valid_raw_message("event-1", "receipt-1")]));
+ queue.fail_delay("receipt-1");
+ let service = TestDeliveryService::new([Ok(WebhookWorkerDisposition::RetryAfter(delay))]);
+ let worker = WebhookEventWorker::new(queue.clone(), service);
+
+ worker.poll_and_process_batch().await;
+
+ assert_eq!(queue.delay_requests(), [("receipt-1".to_string(), delay)]);
+ assert!(queue.delete_requests().is_empty());
+}
+
+#[tokio::test]
+async fn service_failures_do_not_acknowledge_or_stop_the_batch() {
+ let queue = TestQueue::with_receive_result(Ok(vec![
+ valid_raw_message("event-1", "receipt-1"),
+ valid_raw_message("event-2", "receipt-2"),
+ ]));
+ let service = TestDeliveryService::new([
+ Err(TestError("service failed")),
+ Ok(WebhookWorkerDisposition::Acknowledge),
+ ]);
+ let worker = WebhookEventWorker::new(queue.clone(), service.clone());
+
+ worker.poll_and_process_batch().await;
+
+ assert_eq!(
+ service
+ .messages()
+ .into_iter()
+ .map(|message| message.event.event_id)
+ .collect::<Vec<_>>(),
+ ["event-1", "event-2"]
+ );
+ assert_eq!(queue.delete_requests(), ["receipt-2"]);
+ assert!(queue.delay_requests().is_empty());
+}
+
+#[tokio::test]
+async fn processes_each_batch_sequentially() {
+ let operations = Arc::new(Mutex::new(Vec::new()));
+ let queue = TestQueue::with_receive_result(Ok(vec![
+ valid_raw_message("event-1", "receipt-1"),
+ valid_raw_message("event-2", "receipt-2"),
+ ]));
+ queue.set_operations(operations.clone());
+ let service = TestDeliveryService::new([
+ Ok(WebhookWorkerDisposition::Acknowledge),
+ Ok(WebhookWorkerDisposition::Acknowledge),
+ ]);
+ service.set_operations(operations.clone());
+ let worker = WebhookEventWorker::new(queue, service);
+
+ worker.poll_and_process_batch().await;
+
+ assert_eq!(
+ *operations.lock().expect("operations lock"),
+ [
+ "service:event-1",
+ "delete:receipt-1",
+ "service:event-2",
+ "delete:receipt-2",
+ ]
+ );
+}
diff --git a/rust/cloud-storage/webhook/src/lib.rs b/rust/cloud-storage/webhook/src/lib.rs
index 770873f2fa..b409b36db6 100644
--- a/rust/cloud-storage/webhook/src/lib.rs
+++ b/rust/cloud-storage/webhook/src/lib.rs
@@ -1,43 +1,10 @@
#![deny(missing_docs)]
-//! Webhook management hex crate.
-//!
-//! This crate currently contains the core application-layer webhook pieces for
-//! the first iteration:
-//!
-//! - database tables for event ingestion, webhooks, deliveries, and delivery
-//! attempts;
-//! - a typed `filters` model for matching webhook deliveries, shaped as an
-//! array of objects such as
-//! `[{"events": ["document.created"], "ids": ["doc_123"]}]`, where
-//! `ids` is optional and an absent `ids` field matches all entity ids;
-//! - a repository that can create, get, patch, and update webhook validity via
-//! `is_valid`; it can also list active webhooks whose typed `filters` match
-//! an event with
-//! `domain::ports::WebhookRepo::list_active_webhooks_matching_event`;
-//! - inbound handlers for `POST /webhooks`, `PATCH /webhooks/{webhook_id}`, and
-//! `POST /webhooks/{webhook_id}/validate`;
-//! - validation that sends a signed `webhook.validation.test` event and persists
-//! the result in `is_valid`;
-//! - validation attempt rate limiting with the existing `rate_limit` crate using
-//! the key shape `per-user-validate-webhook:{macro_user_id}:{webhook_id}` and
-//! a limit of 10 attempts per 3600-second window;
-//! - a Kafka consumer (`inbound::kafka_consumer`, feature `consumer`) that
-//! subscribes to the `macro.documents` and `macro.channels` topics and hands
-//! every event to the `domain::ingestion::WebhookEventIngestionService`
-//! (feature `ingestion`), whose per-event handlers are still stubs.
-//!
-//! The current implementation intentionally excludes delivery workers, SQS/SNS
-//! infrastructure, redelivery, and auto-pause behavior.
-//!
-//! Temporary limitations: IDs use prefixed UUIDv7 strings (`wh_<uuid_v7>`)
-//! rather than true ULIDs. Custom headers are stored as JSON in `headers`.
-//! Webhook signing secrets are intentionally stored as plaintext in
-//! `signing_secret`, matching the existing bot secret storage approach.
+#![doc = include_str!("../README.md")]
/// Domain models, ports, and services.
pub mod domain;
-#[cfg(any(feature = "inbound", feature = "consumer"))]
-/// HTTP and Kafka consumer adapters.
+#[cfg(any(feature = "inbound", feature = "consumer", feature = "worker"))]
+/// HTTP, Kafka consumer, and queue worker adapters.
pub mod inbound;
#[cfg(feature = "outbound")]
/// Postgres and HTTP adapters.
diff --git a/rust/cloud-storage/webhook/src/outbound.rs b/rust/cloud-storage/webhook/src/outbound.rs
index 6e941a08e0..59044264f5 100644
--- a/rust/cloud-storage/webhook/src/outbound.rs
+++ b/rust/cloud-storage/webhook/src/outbound.rs
@@ -1,7 +1,13 @@
//! Outbound adapters.
+pub mod http_delivery;
pub mod http_validator;
+pub mod pg_delivery_repository;
pub mod pg_repository;
+pub mod sqs_queue;
+pub use http_delivery::ReqwestWebhookDeliveryClient;
pub use http_validator::ReqwestWebhookValidationClient;
+pub use pg_delivery_repository::PgWebhookDeliveryRepository;
pub use pg_repository::PgRepository;
+pub use sqs_queue::SqsWebhookQueue;
diff --git a/rust/cloud-storage/webhook/src/outbound/http_delivery.rs b/rust/cloud-storage/webhook/src/outbound/http_delivery.rs
new file mode 100644
index 0000000000..6ec8f78b50
--- /dev/null
+++ b/rust/cloud-storage/webhook/src/outbound/http_delivery.rs
@@ -0,0 +1,298 @@
+//! Signed HTTP webhook delivery client.
+
+#[cfg(test)]
+#[path = "http_delivery_test.rs"]
+mod http_delivery_test;
+
+use super::http_validator::{
+ EndpointValidationError, ReqwestWebhookValidationClient, is_reserved_macro_header,
+ signature_header, validate_resolved_endpoint_url,
+};
+use crate::domain::{
+ models::{
+ NormalizedWebhookEvent, Webhook, WebhookHeaders, WebhookHttpOutcome,
+ WebhookHttpOutcomeDetails,
+ },
+ ports::WebhookDeliveryClient,
+};
+use reqwest::{
+ Client, Request, Response, StatusCode, Url,
+ header::{CONTENT_TYPE, HeaderMap, HeaderName, HeaderValue},
+};
+use std::time::{Duration, Instant};
+
+const EVENT_HEADER: &str = "x-macro-event";
+const EVENT_ID_HEADER: &str = "x-macro-event-id";
+const TIMESTAMP_HEADER: &str = "x-macro-timestamp";
+const SIGNATURE_HEADER: &str = "x-macro-signature";
+const REDACTED_HEADER_VALUE: &str = "[REDACTED]";
+const RESPONSE_PREVIEW_MAX_BYTES: usize = 4096;
+
+/// Reqwest-backed implementation of [`WebhookDeliveryClient`].
+///
+/// This is the same cloneable, redirect-disabled client used for webhook
+/// validation, so validation and production requests share one security policy.
+pub type ReqwestWebhookDeliveryClient = ReqwestWebhookValidationClient;
+
+impl WebhookDeliveryClient for ReqwestWebhookValidationClient {
+ type Err = anyhow::Error;
+
+ async fn deliver(
+ &self,
+ webhook: &Webhook,
+ event: &NormalizedWebhookEvent,
+ ) -> Result<WebhookHttpOutcome, Self::Err> {
+ let started_at = Instant::now();
+ let raw_body = serde_json::to_vec(&event.broker_envelope)?;
+ let timestamp = chrono::Utc::now().timestamp().to_string();
+ let signature = signature_header(&webhook.signing_secret, &timestamp, &raw_body)?;
+ let headers = match delivery_headers(webhook, event, &timestamp, &signature) {
+ Ok(headers) => headers,
+ Err(message) => {
+ return Ok(WebhookHttpOutcome::PermanentFailure(
+ configuration_failure_details(
+ started_at.elapsed(),
+ "header_configuration",
+ message,
+ ),
+ ));
+ }
+ };
+ let url = match validate_resolved_endpoint_url(
+ &webhook.endpoint_url,
+ self.endpoint_scheme_policy,
+ )
+ .await
+ {
+ Ok(url) => url,
+ Err(error) => return Ok(endpoint_failure(started_at.elapsed(), error)),
+ };
+ let request = match delivery_request(&self.client, url, headers, raw_body) {
+ Ok(request) => request,
+ Err(_) => {
+ return Ok(WebhookHttpOutcome::PermanentFailure(
+ configuration_failure_details(
+ started_at.elapsed(),
+ "request_configuration",
+ "webhook request configuration is invalid",
+ ),
+ ));
+ }
+ };
+
+ let response = match self.client.execute(request).await {
+ Ok(response) => response,
+ Err(error) => {
+ let details = transport_failure_details(started_at.elapsed(), &error);
+ if error.is_builder() {
+ return Ok(WebhookHttpOutcome::PermanentFailure(details));
+ }
+ return Ok(WebhookHttpOutcome::RetryableFailure(details));
+ }
+ };
+
+ Ok(response_outcome(response, started_at).await)
+ }
+}
+
+fn delivery_headers(
+ webhook: &Webhook,
+ event: &NormalizedWebhookEvent,
+ timestamp: &str,
+ signature: &str,
+) -> Result<HeaderMap, &'static str> {
+ if webhook
+ .headers
+ .keys()
+ .any(|name| is_reserved_macro_header(name))
+ {
+ return Err("reserved Macro headers cannot be overridden");
+ }
+
+ let mut headers = HeaderMap::new();
+ for (name, value) in &webhook.headers {
+ let name = HeaderName::from_bytes(name.as_bytes())
+ .map_err(|_| "custom webhook headers are invalid")?;
+ let mut value =
+ HeaderValue::from_str(value).map_err(|_| "custom webhook headers are invalid")?;
+ value.set_sensitive(true);
+ headers.insert(name, value);
+ }
+
+ headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
+ insert_header(&mut headers, EVENT_HEADER, &event.event_name, false)?;
+ insert_header(&mut headers, EVENT_ID_HEADER, &event.event_id, false)?;
+ insert_header(&mut headers, TIMESTAMP_HEADER, timestamp, false)?;
+ insert_header(&mut headers, SIGNATURE_HEADER, signature, true)?;
+
+ Ok(headers)
+}
+
+fn insert_header(
+ headers: &mut HeaderMap,
+ name: &'static str,
+ value: &str,
+ sensitive: bool,
+) -> Result<(), &'static str> {
+ let mut value =
+ HeaderValue::from_str(value).map_err(|_| "webhook event headers are invalid")?;
+ value.set_sensitive(sensitive);
+ headers.insert(HeaderName::from_static(name), value);
+ Ok(())
+}
+
+fn delivery_request(
+ client: &Client,
+ url: Url,
+ headers: HeaderMap,
+ raw_body: Vec<u8>,
+) -> Result<Request, reqwest::Error> {
+ client.post(url).headers(headers).body(raw_body).build()
+}
+
+async fn response_outcome(mut response: Response, started_at: Instant) -> WebhookHttpOutcome {
+ let status = response.status();
+ let response_headers_redacted = Some(redacted_headers(response.headers()));
+ let (response_body_preview, preview_error) = read_response_preview(&mut response).await;
+ let duration = started_at.elapsed();
+
+ let (error_kind, error_message) = match preview_error {
+ Some(error) if error.is_timeout() => (
+ Some("response_timeout".to_string()),
+ Some("webhook response preview timed out".to_string()),
+ ),
+ Some(_) => (
+ Some("response_read".to_string()),
+ Some("webhook response preview could not be read".to_string()),
+ ),
+ None if !status.is_success() => (
+ Some("http_status".to_string()),
+ Some(format!("webhook returned HTTP {}", status.as_u16())),
+ ),
+ None => (None, None),
+ };
+
+ classify_http_status(
+ status,
+ WebhookHttpOutcomeDetails {
+ duration,
+ response_status: Some(status.as_u16()),
+ response_headers_redacted,
+ response_body_preview: Some(response_body_preview),
+ error_kind,
+ error_message,
+ },
+ )
+}
+
+fn classify_http_status(
+ status: StatusCode,
+ details: WebhookHttpOutcomeDetails,
+) -> WebhookHttpOutcome {
+ if status.is_success() {
+ return WebhookHttpOutcome::Success(details);
+ }
+
+ if status == StatusCode::REQUEST_TIMEOUT
+ || status == StatusCode::TOO_MANY_REQUESTS
+ || status.is_server_error()
+ {
+ return WebhookHttpOutcome::RetryableFailure(details);
+ }
+
+ WebhookHttpOutcome::PermanentFailure(details)
+}
+
+async fn read_response_preview(response: &mut Response) -> (String, Option<reqwest::Error>) {
+ let mut preview = Vec::new();
+ while preview.len() < RESPONSE_PREVIEW_MAX_BYTES {
+ match response.chunk().await {
+ Ok(Some(chunk)) => {
+ let remaining = RESPONSE_PREVIEW_MAX_BYTES - preview.len();
+ preview.extend_from_slice(&chunk[..chunk.len().min(remaining)]);
+ }
+ Ok(None) => break,
+ Err(error) => return (utf8_lossy_preview(&preview), Some(error)),
+ }
+ }
+
+ (utf8_lossy_preview(&preview), None)
+}
+
+fn utf8_lossy_preview(bytes: &[u8]) -> String {
+ let bytes = &bytes[..bytes.len().min(RESPONSE_PREVIEW_MAX_BYTES)];
+ let mut preview = String::from_utf8_lossy(bytes).into_owned();
+ if preview.len() > RESPONSE_PREVIEW_MAX_BYTES {
+ let mut end = RESPONSE_PREVIEW_MAX_BYTES;
+ while !preview.is_char_boundary(end) {
+ end -= 1;
+ }
+ preview.truncate(end);
+ }
+ preview
+}
+
+fn redacted_headers(headers: &HeaderMap) -> WebhookHeaders {
+ headers
+ .keys()
+ .map(|name| (name.as_str().to_string(), REDACTED_HEADER_VALUE.to_string()))
+ .collect()
+}
+
+fn endpoint_failure(duration: Duration, error: EndpointValidationError) -> WebhookHttpOutcome {
+ let details = configuration_failure_details(
+ duration,
+ if error.is_retryable() {
+ "dns_resolution"
+ } else {
+ "endpoint_configuration"
+ },
+ error.message(),
+ );
+
+ if error.is_retryable() {
+ WebhookHttpOutcome::RetryableFailure(details)
+ } else {
+ WebhookHttpOutcome::PermanentFailure(details)
+ }
+}
+
+fn configuration_failure_details(
+ duration: Duration,
+ error_kind: &str,
+ error_message: &str,
+) -> WebhookHttpOutcomeDetails {
+ WebhookHttpOutcomeDetails {
+ duration,
+ response_status: None,
+ response_headers_redacted: None,
+ response_body_preview: None,
+ error_kind: Some(error_kind.to_string()),
+ error_message: Some(error_message.to_string()),
+ }
+}
+
+fn transport_failure_details(
+ duration: Duration,
+ error: &reqwest::Error,
+) -> WebhookHttpOutcomeDetails {
+ let (error_kind, error_message) = if error.is_timeout() {
+ ("timeout", "webhook delivery timed out after 5 seconds")
+ } else if error.is_builder() {
+ (
+ "request_configuration",
+ "webhook request configuration is invalid",
+ )
+ } else {
+ ("network", "webhook endpoint could not be reached")
+ };
+
+ WebhookHttpOutcomeDetails {
+ duration,
+ response_status: None,
+ response_headers_redacted: None,
+ response_body_preview: None,
+ error_kind: Some(error_kind.to_string()),
+ error_message: Some(error_message.to_string()),
+ }
+}
diff --git a/rust/cloud-storage/webhook/src/outbound/http_delivery_test.rs b/rust/cloud-storage/webhook/src/outbound/http_delivery_test.rs
new file mode 100644
index 0000000000..9752e497a4
--- /dev/null
+++ b/rust/cloud-storage/webhook/src/outbound/http_delivery_test.rs
@@ -0,0 +1,254 @@
+use super::*;
+use crate::domain::{
+ models::{WebhookFilter, WebhookStatus},
+ ports::WebhookDeliveryClient,
+};
+use chrono::Utc;
+use serde_json::json;
+use std::collections::BTreeMap;
+
+fn webhook(endpoint_url: &str, headers: WebhookHeaders) -> Webhook {
+ Webhook {
+ id: "wh_test".to_string(),
+ workspace_id: "workspace_test".to_string(),
+ name: "Test webhook".to_string(),
+ endpoint_url: endpoint_url.to_string(),
+ signing_secret: "signing-secret".to_string(),
+ headers,
+ status: WebhookStatus::Active,
+ is_valid: true,
+ created_by_user_id: "macro|[email protected]".to_string(),
+ created_at: Utc::now(),
+ updated_at: Utc::now(),
+ deleted_at: None,
+ filters: vec![WebhookFilter {
+ events: vec!["document.created".to_string()],
+ ids: None,
+ }],
+ }
+}
+
+fn event() -> NormalizedWebhookEvent {
+ NormalizedWebhookEvent {
+ event_id: "evt_test".to_string(),
+ schema_version: 1,
+ event_name: "document.created".to_string(),
+ entity_type: "document".to_string(),
+ entity_id: "doc_test".to_string(),
+ ordering_key: "doc_test".to_string(),
+ occurred_at: Utc::now(),
+ broker_envelope: json!({
+ "event_id": "evt_test",
+ "schema_version": 1,
+ "event_type": "document.created",
+ "metadata": {"document_id": "doc_test"},
+ }),
+ }
+}
+
+fn outcome_details() -> WebhookHttpOutcomeDetails {
+ WebhookHttpOutcomeDetails {
+ duration: Duration::from_millis(10),
+ response_status: None,
+ response_headers_redacted: None,
+ response_body_preview: None,
+ error_kind: None,
+ error_message: None,
+ }
+}
+
+#[test]
+fn builds_signed_request_with_exact_broker_envelope() {
+ let webhook = webhook(
+ "https://example.com/webhook",
+ BTreeMap::from([("Authorization".to_string(), "Bearer secret".to_string())]),
+ );
+ let event = event();
+ let raw_body = serde_json::to_vec(&event.broker_envelope).unwrap();
+ let timestamp = "1783699200";
+ let signature = signature_header(&webhook.signing_secret, timestamp, &raw_body).unwrap();
+ let headers = delivery_headers(&webhook, &event, timestamp, &signature).unwrap();
+ let request = delivery_request(
+ &Client::new(),
+ Url::parse(&webhook.endpoint_url).unwrap(),
+ headers,
+ raw_body.clone(),
+ )
+ .unwrap();
+
+ assert_eq!(
+ request.body().and_then(reqwest::Body::as_bytes),
+ Some(raw_body.as_slice())
+ );
+ assert_eq!(request.headers()[CONTENT_TYPE], "application/json");
+ assert_eq!(request.headers()[EVENT_HEADER], event.event_name);
+ assert_eq!(request.headers()[EVENT_ID_HEADER], event.event_id);
+ assert_eq!(request.headers()[TIMESTAMP_HEADER], timestamp);
+ assert_eq!(request.headers()[SIGNATURE_HEADER], signature);
+ assert_eq!(request.headers()["authorization"], "Bearer secret");
+ assert!(request.headers()[SIGNATURE_HEADER].is_sensitive());
+ assert!(request.headers()["authorization"].is_sensitive());
+}
+
+#[test]
+fn rejects_reserved_headers_case_insensitively() {
+ for name in [
+ "X-Macro-Event",
+ "x-macro-event-id",
+ "X-MACRO-TIMESTAMP",
+ "x-Macro-Signature",
+ ] {
+ let webhook = webhook(
+ "https://example.com/webhook",
+ BTreeMap::from([(name.to_string(), "override".to_string())]),
+ );
+
+ assert_eq!(
+ delivery_headers(&webhook, &event(), "123", "v1=signature"),
+ Err("reserved Macro headers cannot be overridden")
+ );
+ }
+}
+
+#[test]
+fn rejects_invalid_custom_header_names_and_values() {
+ let invalid_name = webhook(
+ "https://example.com/webhook",
+ BTreeMap::from([("bad header".to_string(), "value".to_string())]),
+ );
+ let invalid_value = webhook(
+ "https://example.com/webhook",
+ BTreeMap::from([("X-Custom".to_string(), "first\nsecond".to_string())]),
+ );
+
+ assert_eq!(
+ delivery_headers(&invalid_name, &event(), "123", "v1=signature"),
+ Err("custom webhook headers are invalid")
+ );
+ assert_eq!(
+ delivery_headers(&invalid_value, &event(), "123", "v1=signature"),
+ Err("custom webhook headers are invalid")
+ );
+}
+
+#[test]
+fn always_uses_json_content_type() {
+ let webhook = webhook(
+ "https://example.com/webhook",
+ BTreeMap::from([("content-type".to_string(), "text/plain".to_string())]),
+ );
+
+ let headers = delivery_headers(&webhook, &event(), "123", "v1=signature").unwrap();
+
+ assert_eq!(headers[CONTENT_TYPE], "application/json");
+}
+
+#[test]
+fn classifies_http_response_statuses() {
+ for status in [200, 201, 204, 299] {
+ assert!(matches!(
+ classify_http_status(StatusCode::from_u16(status).unwrap(), outcome_details()),
+ WebhookHttpOutcome::Success(_)
+ ));
+ }
+
+ for status in [408, 429, 500, 503, 599] {
+ assert!(matches!(
+ classify_http_status(StatusCode::from_u16(status).unwrap(), outcome_details()),
+ WebhookHttpOutcome::RetryableFailure(_)
+ ));
+ }
+
+ for status in [300, 400, 401, 404, 409, 422] {
+ assert!(matches!(
+ classify_http_status(StatusCode::from_u16(status).unwrap(), outcome_details()),
+ WebhookHttpOutcome::PermanentFailure(_)
+ ));
+ }
+}
+
+#[test]
+fn redacts_response_header_values() {
+ let mut headers = HeaderMap::new();
+ headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
+ headers.insert(
+ HeaderName::from_static("x-request-id"),
+ HeaderValue::from_static("private-request-id"),
+ );
+
+ assert_eq!(
+ redacted_headers(&headers),
+ BTreeMap::from([
+ (
+ "content-type".to_string(),
+ REDACTED_HEADER_VALUE.to_string()
+ ),
+ (
+ "x-request-id".to_string(),
+ REDACTED_HEADER_VALUE.to_string()
+ ),
+ ])
+ );
+}
+
+#[test]
+fn caps_utf8_lossy_response_preview_at_4096_bytes() {
+ let oversized = vec![b'a'; RESPONSE_PREVIEW_MAX_BYTES + 100];
+ let invalid_utf8 = vec![0xff; RESPONSE_PREVIEW_MAX_BYTES];
+
+ let preview = utf8_lossy_preview(&oversized);
+ let lossy_preview = utf8_lossy_preview(&invalid_utf8);
+
+ assert_eq!(preview.len(), RESPONSE_PREVIEW_MAX_BYTES);
+ assert!(lossy_preview.contains('\u{fffd}'));
+ assert!(lossy_preview.len() <= RESPONSE_PREVIEW_MAX_BYTES);
+}
+
+#[tokio::test]
+async fn invalid_endpoint_configuration_is_permanent() {
+ let client = ReqwestWebhookDeliveryClient::new().unwrap();
+ let outcome = client
+ .deliver(
+ &webhook("http://example.com/webhook", BTreeMap::new()),
+ &event(),
+ )
+ .await
+ .unwrap();
+
+ let WebhookHttpOutcome::PermanentFailure(details) = outcome else {
+ panic!("invalid endpoint should be permanent");
+ };
+ assert_eq!(details.response_status, None);
+ assert_eq!(
+ details.error_kind.as_deref(),
+ Some("endpoint_configuration")
+ );
+ assert_eq!(
+ details.error_message.as_deref(),
+ Some("webhook endpoint URL must use HTTPS")
+ );
+}
+
+#[tokio::test]
+async fn reserved_header_configuration_is_permanent_without_dns_lookup() {
+ let client = ReqwestWebhookDeliveryClient::new().unwrap();
+ let outcome = client
+ .deliver(
+ &webhook(
+ "https://unresolvable.invalid/webhook",
+ BTreeMap::from([("X-Macro-Signature".to_string(), "private".to_string())]),
+ ),
+ &event(),
+ )
+ .await
+ .unwrap();
+
+ let WebhookHttpOutcome::PermanentFailure(details) = outcome else {
+ panic!("reserved header should be permanent");
+ };
+ assert_eq!(details.error_kind.as_deref(), Some("header_configuration"));
+ assert_eq!(
+ details.error_message.as_deref(),
+ Some("reserved Macro headers cannot be overridden")
+ );
+}
diff --git a/rust/cloud-storage/webhook/src/outbound/http_validator.rs b/rust/cloud-storage/webhook/src/outbound/http_validator.rs
index 592387ee4f..2d78309b0a 100644
--- a/rust/cloud-storage/webhook/src/outbound/http_validator.rs
+++ b/rust/cloud-storage/webhook/src/outbound/http_validator.rs
@@ -5,7 +5,7 @@
mod http_validator_test;
use crate::domain::{
- models::{Webhook, WebhookValidationResult},
+ models::{Webhook, WebhookEndpointSchemePolicy, WebhookValidationResult},
ports::WebhookValidationClient,
};
use hmac::{Hmac, Mac};
@@ -16,25 +16,36 @@ use std::{net::IpAddr, time::Duration};
use tokio::net::lookup_host;
const EVENT_NAME: &str = "webhook.validation.test";
-const REQUEST_TIMEOUT: Duration = Duration::from_secs(5);
+pub(super) const REQUEST_TIMEOUT: Duration = Duration::from_secs(5);
type HmacSha256 = Hmac<Sha256>;
/// Reqwest-backed implementation of [`WebhookValidationClient`].
#[derive(Clone)]
pub struct ReqwestWebhookValidationClient {
- client: Client,
+ pub(super) client: Client,
+ pub(super) endpoint_scheme_policy: WebhookEndpointSchemePolicy,
}
impl ReqwestWebhookValidationClient {
- /// Create a validation client with the required webhook delivery limits.
+ /// Create a validation client that requires public HTTPS endpoints.
pub fn new() -> Result<Self, reqwest::Error> {
+ Self::new_with_endpoint_scheme_policy(WebhookEndpointSchemePolicy::HttpsOnly)
+ }
+
+ /// Create a validation client with an explicit endpoint policy.
+ pub fn new_with_endpoint_scheme_policy(
+ endpoint_scheme_policy: WebhookEndpointSchemePolicy,
+ ) -> Result<Self, reqwest::Error> {
let client = Client::builder()
.timeout(REQUEST_TIMEOUT)
.redirect(Policy::none())
.build()?;
- Ok(Self { client })
+ Ok(Self {
+ client,
+ endpoint_scheme_policy,
+ })
}
}
@@ -45,9 +56,14 @@ impl WebhookValidationClient for ReqwestWebhookValidationClient {
&self,
webhook: Webhook,
) -> Result<WebhookValidationResult, Self::Err> {
- let url = match validate_resolved_endpoint_url(&webhook.endpoint_url).await {
+ let url = match validate_resolved_endpoint_url(
+ &webhook.endpoint_url,
+ self.endpoint_scheme_policy,
+ )
+ .await
+ {
Ok(url) => url,
- Err(message) => return Ok(failure(None, message)),
+ Err(error) => return Ok(failure(None, error.message())),
};
let event_id = new_validation_event_id();
@@ -107,7 +123,7 @@ fn validation_body(webhook_id: &str, event_id: &str) -> Result<Vec<u8>, serde_js
}))
}
-fn signature_header(
+pub(super) fn signature_header(
secret: &str,
timestamp: &str,
raw_body: &[u8],
@@ -119,45 +135,96 @@ fn signature_header(
Ok(format!("v1={}", hex::encode(mac.finalize().into_bytes())))
}
-async fn validate_resolved_endpoint_url(value: &str) -> Result<Url, &'static str> {
- let url = validate_endpoint_url(value)?;
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) enum EndpointValidationError {
+ InvalidConfiguration(&'static str),
+ ResolutionFailure(&'static str),
+}
+
+impl EndpointValidationError {
+ pub(super) const fn message(self) -> &'static str {
+ match self {
+ Self::InvalidConfiguration(message) | Self::ResolutionFailure(message) => message,
+ }
+ }
+
+ pub(super) const fn is_retryable(self) -> bool {
+ matches!(self, Self::ResolutionFailure(_))
+ }
+}
+
+pub(super) async fn validate_resolved_endpoint_url(
+ value: &str,
+ endpoint_scheme_policy: WebhookEndpointSchemePolicy,
+) -> Result<Url, EndpointValidationError> {
+ let url = validate_endpoint_url(value, endpoint_scheme_policy)?;
let Some(host) = url.host_str().map(str::to_owned) else {
- return Err("webhook endpoint URL host is invalid");
+ return Err(EndpointValidationError::InvalidConfiguration(
+ "webhook endpoint URL host is invalid",
+ ));
};
let Some(port) = url.port_or_known_default() else {
- return Err("webhook endpoint URL port is invalid");
+ return Err(EndpointValidationError::InvalidConfiguration(
+ "webhook endpoint URL port is invalid",
+ ));
};
- let mut resolved = lookup_host((host.as_str(), port))
- .await
- .map_err(|_| "webhook endpoint host could not be resolved")?;
+ let mut resolved =
+ match tokio::time::timeout(REQUEST_TIMEOUT, lookup_host((host.as_str(), port))).await {
+ Ok(Ok(resolved)) => resolved,
+ Ok(Err(_)) => {
+ return Err(EndpointValidationError::ResolutionFailure(
+ "webhook endpoint host could not be resolved",
+ ));
+ }
+ Err(_) => {
+ return Err(EndpointValidationError::ResolutionFailure(
+ "webhook endpoint host resolution timed out",
+ ));
+ }
+ };
let mut saw_address = false;
for address in resolved.by_ref() {
saw_address = true;
- if is_blocked_ip(address.ip()) {
- return Err("webhook endpoint host resolves to a disallowed address");
+ if !endpoint_scheme_policy.allows_local_addresses() && is_blocked_ip(address.ip()) {
+ return Err(EndpointValidationError::InvalidConfiguration(
+ "webhook endpoint host resolves to a disallowed address",
+ ));
}
}
if !saw_address {
- return Err("webhook endpoint host could not be resolved");
+ return Err(EndpointValidationError::ResolutionFailure(
+ "webhook endpoint host could not be resolved",
+ ));
}
Ok(url)
}
-fn validate_endpoint_url(value: &str) -> Result<Url, &'static str> {
- let url = Url::parse(value).map_err(|_| "webhook endpoint URL is invalid")?;
- if url.scheme() != "https" {
- return Err("webhook endpoint URL must use HTTPS");
+fn validate_endpoint_url(
+ value: &str,
+ endpoint_scheme_policy: WebhookEndpointSchemePolicy,
+) -> Result<Url, EndpointValidationError> {
+ let url = Url::parse(value).map_err(|_| {
+ EndpointValidationError::InvalidConfiguration("webhook endpoint URL is invalid")
+ })?;
+ if !endpoint_scheme_policy.allows(url.scheme()) {
+ return Err(EndpointValidationError::InvalidConfiguration(
+ "webhook endpoint URL must use HTTPS",
+ ));
}
let Some(host) = url.host_str() else {
- return Err("webhook endpoint URL host is invalid");
+ return Err(EndpointValidationError::InvalidConfiguration(
+ "webhook endpoint URL host is invalid",
+ ));
};
- if is_blocked_host(host) {
- return Err("webhook endpoint host is not allowed");
+ if !endpoint_scheme_policy.allows_local_addresses() && is_blocked_host(host) {
+ return Err(EndpointValidationError::InvalidConfiguration(
+ "webhook endpoint host is not allowed",
+ ));
}
Ok(url)
@@ -191,7 +258,7 @@ fn is_blocked_ip(ip: IpAddr) -> bool {
}
}
-fn is_reserved_macro_header(name: &str) -> bool {
+pub(super) fn is_reserved_macro_header(name: &str) -> bool {
matches!(
name.to_ascii_lowercase().as_str(),
"x-macro-event" | "x-macro-event-id" | "x-macro-timestamp" | "x-macro-signature"
diff --git a/rust/cloud-storage/webhook/src/outbound/http_validator_test.rs b/rust/cloud-storage/webhook/src/outbound/http_validator_test.rs
index 20bce9cb95..15f85860fe 100644
--- a/rust/cloud-storage/webhook/src/outbound/http_validator_test.rs
+++ b/rust/cloud-storage/webhook/src/outbound/http_validator_test.rs
@@ -1,5 +1,10 @@
use super::*;
+#[test]
+fn uses_five_second_request_timeout() {
+ assert_eq!(REQUEST_TIMEOUT, Duration::from_secs(5));
+}
+
#[test]
fn builds_deterministic_validation_body() {
let body = validation_body("wh_test", "evt_test").expect("body should serialize");
@@ -33,28 +38,66 @@ fn signs_timestamp_and_raw_body() {
#[test]
fn rejects_non_https_endpoint() {
- let error = validate_endpoint_url("http://example.com/webhook").expect_err("http is invalid");
+ let error = validate_endpoint_url(
+ "http://example.com/webhook",
+ WebhookEndpointSchemePolicy::HttpsOnly,
+ )
+ .expect_err("http is invalid");
+
+ assert_eq!(error.message(), "webhook endpoint URL must use HTTPS");
+}
- assert_eq!(error, "webhook endpoint URL must use HTTPS");
+#[test]
+fn allows_local_http_endpoints_when_policy_permits_them() {
+ let policy = WebhookEndpointSchemePolicy::HttpAndHttps;
+
+ for value in [
+ "http://localhost/webhook",
+ "http://127.0.0.1/webhook",
+ "http://10.1.2.3/webhook",
+ "http://[::1]/webhook",
+ "http://[fe80::1]/webhook",
+ ] {
+ let url = validate_endpoint_url(value, policy).expect("local endpoint should be allowed");
+
+ assert_eq!(url.scheme(), "http");
+ }
}
#[test]
fn rejects_localhost_and_private_ips() {
- assert!(validate_endpoint_url("https://localhost/webhook").is_err());
- assert!(validate_endpoint_url("https://127.0.0.1/webhook").is_err());
- assert!(validate_endpoint_url("https://10.1.2.3/webhook").is_err());
- assert!(validate_endpoint_url("https://169.254.169.254/latest/meta-data").is_err());
- assert!(validate_endpoint_url("https://[::1]/webhook").is_err());
- assert!(validate_endpoint_url("https://[fe80::1]/webhook").is_err());
+ let policy = WebhookEndpointSchemePolicy::HttpsOnly;
+
+ assert!(validate_endpoint_url("https://localhost/webhook", policy).is_err());
+ assert!(validate_endpoint_url("https://127.0.0.1/webhook", policy).is_err());
+ assert!(validate_endpoint_url("https://10.1.2.3/webhook", policy).is_err());
+ assert!(validate_endpoint_url("https://169.254.169.254/latest/meta-data", policy).is_err());
+ assert!(validate_endpoint_url("https://[::1]/webhook", policy).is_err());
+ assert!(validate_endpoint_url("https://[fe80::1]/webhook", policy).is_err());
}
#[tokio::test]
async fn rejects_hosts_that_resolve_to_blocked_addresses() {
- let error = validate_resolved_endpoint_url("https://localhost/webhook")
- .await
- .expect_err("localhost is invalid");
+ let error = validate_resolved_endpoint_url(
+ "https://localhost/webhook",
+ WebhookEndpointSchemePolicy::HttpsOnly,
+ )
+ .await
+ .expect_err("localhost is invalid");
+
+ assert_eq!(error.message(), "webhook endpoint host is not allowed");
+}
- assert_eq!(error, "webhook endpoint host is not allowed");
+#[tokio::test]
+async fn allows_localhost_resolution_when_policy_permits_it() {
+ let url = validate_resolved_endpoint_url(
+ "http://localhost/webhook",
+ WebhookEndpointSchemePolicy::HttpAndHttps,
+ )
+ .await
+ .expect("localhost should resolve");
+
+ assert_eq!(url.host_str(), Some("localhost"));
}
#[test]
@@ -66,7 +109,11 @@ fn blocks_ipv6_link_local_addresses() {
#[test]
fn accepts_public_https_endpoint() {
- let url = validate_endpoint_url("https://example.com/webhook").expect("public https is valid");
+ let url = validate_endpoint_url(
+ "https://example.com/webhook",
+ WebhookEndpointSchemePolicy::HttpsOnly,
+ )
+ .expect("public https is valid");
assert_eq!(url.host_str(), Some("example.com"));
}
diff --git a/rust/cloud-storage/webhook/src/outbound/pg_delivery_repository.rs b/rust/cloud-storage/webhook/src/outbound/pg_delivery_repository.rs
new file mode 100644
index 0000000000..9c20be26b6
--- /dev/null
+++ b/rust/cloud-storage/webhook/src/outbound/pg_delivery_repository.rs
@@ -0,0 +1,655 @@
+//! PostgreSQL webhook delivery repository.
+
+#[cfg(test)]
+#[path = "pg_delivery_repository_test.rs"]
+mod pg_delivery_repository_test;
+
+use crate::domain::{
+ models::{
+ PreparedWebhookDelivery, Webhook, WebhookDeliveryAttempt, WebhookDeliveryAttemptStatus,
+ WebhookDeliveryStatus, WebhookEventQueueMessage, WebhookFilters, WebhookHeaders,
+ WebhookHttpOutcomeDetails, WebhookStatus,
+ },
+ ports::WebhookDeliveryRepository,
+};
+use chrono::{DateTime, Utc};
+use serde_json::Value;
+use sqlx::{PgPool, Postgres, Transaction};
+use std::str::FromStr;
+
+const REDACTED_HEADER_VALUE: &str = "[REDACTED]";
+
+/// PostgreSQL-backed implementation of [`WebhookDeliveryRepository`].
+#[derive(Clone)]
+pub struct PgWebhookDeliveryRepository {
+ pool: PgPool,
+}
+
+impl PgWebhookDeliveryRepository {
+ /// Create a webhook delivery repository backed by the supplied pool.
+ pub fn new(pool: PgPool) -> Self {
+ Self { pool }
+ }
+}
+
+#[derive(sqlx::FromRow)]
+struct PreparedDeliveryRow {
+ delivery_id: String,
+ delivery_status: String,
+ attempt_count: i32,
+ next_attempt_at: Option<DateTime<Utc>>,
+ webhook_id: String,
+ workspace_id: String,
+ name: String,
+ endpoint_url: String,
+ signing_secret: String,
+ headers: Value,
+ webhook_status: String,
+ is_valid: bool,
+ created_by_user_id: String,
+ created_at: DateTime<Utc>,
+ updated_at: DateTime<Utc>,
+ deleted_at: Option<DateTime<Utc>>,
+ filters: Value,
+}
+
+#[derive(Clone, Copy)]
+struct CompletionState {
+ attempt_status: WebhookDeliveryAttemptStatus,
+ delivery_status: WebhookDeliveryStatus,
+ next_attempt_at: Option<DateTime<Utc>>,
+ delivered: bool,
+}
+
+fn new_delivery_id() -> String {
+ format!("whd_{}", macro_uuid::generate_uuid_v7())
+}
+
+fn new_attempt_id() -> String {
+ format!("wha_{}", macro_uuid::generate_uuid_v7())
+}
+
+fn decode_error(
+ column: &str,
+ message: impl Into<Box<dyn std::error::Error + Send + Sync>>,
+) -> sqlx::Error {
+ sqlx::Error::ColumnDecode {
+ index: column.to_string(),
+ source: message.into(),
+ }
+}
+
+fn row_to_prepared_delivery(
+ row: PreparedDeliveryRow,
+) -> Result<PreparedWebhookDelivery, sqlx::Error> {
+ let status = WebhookDeliveryStatus::from_str(&row.delivery_status)
+ .map_err(|message| decode_error("delivery_status", message))?;
+ let webhook_status = WebhookStatus::from_str(&row.webhook_status)
+ .map_err(|message| decode_error("webhook_status", message))?;
+ let filters = serde_json::from_value::<WebhookFilters>(row.filters)
+ .map_err(|error| decode_error("filters", error))?;
+ let headers = serde_json::from_value::<WebhookHeaders>(row.headers)
+ .map_err(|error| decode_error("headers", error))?;
+ let attempt_count =
+ u32::try_from(row.attempt_count).map_err(|error| decode_error("attempt_count", error))?;
+
+ Ok(PreparedWebhookDelivery {
+ delivery_id: row.delivery_id,
+ webhook: Webhook {
+ id: row.webhook_id,
+ workspace_id: row.workspace_id,
+ name: row.name,
+ endpoint_url: row.endpoint_url,
+ signing_secret: row.signing_secret,
+ headers,
+ status: webhook_status,
+ is_valid: row.is_valid,
+ created_by_user_id: row.created_by_user_id,
+ created_at: row.created_at,
+ updated_at: row.updated_at,
+ deleted_at: row.deleted_at,
+ filters,
+ },
+ status,
+ attempt_count,
+ next_attempt_at: row.next_attempt_at,
+ })
+}
+
+fn encode_error(error: impl std::error::Error + Send + Sync + 'static) -> sqlx::Error {
+ sqlx::Error::Encode(Box::new(error))
+}
+
+fn duration_milliseconds(details: &WebhookHttpOutcomeDetails) -> Result<i32, sqlx::Error> {
+ i32::try_from(details.duration.as_millis()).map_err(encode_error)
+}
+
+fn response_status(details: &WebhookHttpOutcomeDetails) -> Option<i32> {
+ details.response_status.map(i32::from)
+}
+
+fn response_headers(details: &WebhookHttpOutcomeDetails) -> Result<Option<Value>, sqlx::Error> {
+ details
+ .response_headers_redacted
+ .as_ref()
+ .map(serde_json::to_value)
+ .transpose()
+ .map_err(encode_error)
+}
+
+async fn interrupt_current_attempt(
+ transaction: &mut Transaction<'_, Postgres>,
+ delivery_id: &str,
+ webhook_id: &str,
+ attempt_count: i32,
+) -> Result<(), sqlx::Error> {
+ if attempt_count == 0 {
+ return Ok(());
+ }
+
+ let interrupted_status = WebhookDeliveryAttemptStatus::Interrupted.as_str();
+ let in_progress_status = WebhookDeliveryAttemptStatus::InProgress.as_str();
+ let interrupted = sqlx::query!(
+ r#"
+ UPDATE webhook_delivery_attempt
+ SET
+ status = $3,
+ completed_at = now()
+ WHERE webhook_delivery_id = $1
+ AND attempt_number = $2
+ AND status = $4
+ RETURNING completed_at AS "completed_at!"
+ "#,
+ delivery_id,
+ attempt_count,
+ interrupted_status,
+ in_progress_status
+ )
+ .fetch_optional(&mut **transaction)
+ .await?;
+
+ if let Some(interrupted) = interrupted {
+ sqlx::query!(
+ r#"
+ UPDATE webhook
+ SET
+ last_failure_at = GREATEST(COALESCE(last_failure_at, $2), $2),
+ updated_at = GREATEST(updated_at, $2)
+ WHERE id = $1
+ "#,
+ webhook_id,
+ interrupted.completed_at
+ )
+ .execute(&mut **transaction)
+ .await?;
+ }
+
+ Ok(())
+}
+
+async fn complete_attempt(
+ pool: &PgPool,
+ attempt: &WebhookDeliveryAttempt,
+ details: &WebhookHttpOutcomeDetails,
+ completion: CompletionState,
+) -> Result<(), sqlx::Error> {
+ let attempt_number = i32::try_from(attempt.attempt_number).map_err(encode_error)?;
+ let duration_ms = duration_milliseconds(details)?;
+ let response_status = response_status(details);
+ let response_headers_redacted = response_headers(details)?;
+ let mut transaction = pool.begin().await?;
+
+ let delivery = sqlx::query!(
+ r#"
+ SELECT webhook_id, status, attempt_count
+ FROM webhook_delivery
+ WHERE id = $1
+ FOR UPDATE
+ "#,
+ attempt.delivery_id
+ )
+ .fetch_optional(&mut *transaction)
+ .await?
+ .ok_or(sqlx::Error::RowNotFound)?;
+
+ if delivery.status != WebhookDeliveryStatus::InProgress.as_str()
+ || delivery.attempt_count != attempt_number
+ {
+ return Err(sqlx::Error::RowNotFound);
+ }
+
+ let attempt_status = completion.attempt_status.as_str();
+ let in_progress_attempt_status = WebhookDeliveryAttemptStatus::InProgress.as_str();
+ let completed = sqlx::query!(
+ r#"
+ UPDATE webhook_delivery_attempt
+ SET
+ status = $4,
+ completed_at = now(),
+ duration_ms = $5,
+ response_status = $6,
+ response_headers_redacted = $7,
+ response_body_preview = $8,
+ error_kind = $9,
+ error_message = $10
+ WHERE id = $1
+ AND webhook_delivery_id = $2
+ AND attempt_number = $3
+ AND status = $11
+ RETURNING completed_at AS "completed_at!"
+ "#,
+ attempt.attempt_id,
+ attempt.delivery_id,
+ attempt_number,
+ attempt_status,
+ duration_ms,
+ response_status,
+ response_headers_redacted,
+ details.response_body_preview,
+ details.error_kind,
+ details.error_message,
+ in_progress_attempt_status
+ )
+ .fetch_optional(&mut *transaction)
+ .await?
+ .ok_or(sqlx::Error::RowNotFound)?;
+
+ let delivery_status = completion.delivery_status.as_str();
+ let delivered_at = completion.delivered.then_some(completed.completed_at);
+ let updated = sqlx::query!(
+ r#"
+ UPDATE webhook_delivery
+ SET
+ status = $2,
+ next_attempt_at = $3,
+ delivered_at = $4,
+ updated_at = $5
+ WHERE id = $1
+ AND status = $6
+ AND attempt_count = $7
+ "#,
+ attempt.delivery_id,
+ delivery_status,
+ completion.next_attempt_at,
+ delivered_at,
+ completed.completed_at,
+ WebhookDeliveryStatus::InProgress.as_str(),
+ attempt_number
+ )
+ .execute(&mut *transaction)
+ .await?;
+
+ if updated.rows_affected() != 1 {
+ return Err(sqlx::Error::RowNotFound);
+ }
+
+ if completion.delivered {
+ sqlx::query!(
+ r#"
+ UPDATE webhook
+ SET
+ last_success_at = GREATEST(COALESCE(last_success_at, $2), $2),
+ updated_at = GREATEST(updated_at, $2)
+ WHERE id = $1
+ "#,
+ delivery.webhook_id,
+ completed.completed_at
+ )
+ .execute(&mut *transaction)
+ .await?;
+ } else {
+ sqlx::query!(
+ r#"
+ UPDATE webhook
+ SET
+ last_failure_at = GREATEST(COALESCE(last_failure_at, $2), $2),
+ updated_at = GREATEST(updated_at, $2)
+ WHERE id = $1
+ "#,
+ delivery.webhook_id,
+ completed.completed_at
+ )
+ .execute(&mut *transaction)
+ .await?;
+ }
+
+ transaction.commit().await
+}
+
+impl WebhookDeliveryRepository for PgWebhookDeliveryRepository {
+ type Err = sqlx::Error;
+
+ #[tracing::instrument(skip(self, message), fields(webhook_id = %message.webhook_id, event_id = %message.event.event_id), err)]
+ async fn prepare_delivery(
+ &self,
+ message: &WebhookEventQueueMessage,
+ ) -> Result<Option<PreparedWebhookDelivery>, Self::Err> {
+ let mut transaction = self.pool.begin().await?;
+ let delivery_id = new_delivery_id();
+ let delivery_status = WebhookDeliveryStatus::Queued.as_str();
+ let event_schema_version = i32::from(message.event.schema_version);
+
+ sqlx::query!(
+ r#"
+ INSERT INTO webhook_delivery (
+ id,
+ webhook_id,
+ event_id,
+ event,
+ event_schema_version,
+ event_occurred_at,
+ event_entity_type,
+ event_entity_id,
+ event_ordering_key,
+ status,
+ request_url,
+ request_headers_redacted,
+ request_body
+ )
+ SELECT
+ $1,
+ w.id,
+ $3,
+ $4,
+ $5,
+ $6,
+ $7,
+ $8,
+ $9,
+ $10,
+ w.endpoint_url,
+ COALESCE(
+ (
+ SELECT jsonb_object_agg(header_name, $11::TEXT)
+ FROM jsonb_object_keys(w.headers) AS header_name
+ ),
+ '{}'::JSONB
+ ),
+ $12
+ FROM webhook w
+ WHERE w.id = $2
+ ON CONFLICT (webhook_id, event_id) DO NOTHING
+ "#,
+ delivery_id,
+ message.webhook_id,
+ message.event.event_id,
+ message.event.event_name,
+ event_schema_version,
+ message.event.occurred_at,
+ message.event.entity_type,
+ message.event.entity_id,
+ message.event.ordering_key,
+ delivery_status,
+ REDACTED_HEADER_VALUE,
+ message.event.broker_envelope
+ )
+ .execute(&mut *transaction)
+ .await?;
+
+ let row = sqlx::query_as!(
+ PreparedDeliveryRow,
+ r#"
+ SELECT
+ d.id AS delivery_id,
+ d.status AS delivery_status,
+ d.attempt_count,
+ d.next_attempt_at,
+ w.id AS webhook_id,
+ w.workspace_id,
+ w.name,
+ w.endpoint_url,
+ w.signing_secret,
+ w.headers,
+ w.status AS webhook_status,
+ w.is_valid,
+ w.created_by_user_id,
+ w.created_at,
+ w.updated_at,
+ w.deleted_at,
+ w.filters
+ FROM webhook_delivery d
+ JOIN webhook w ON w.id = d.webhook_id
+ WHERE d.webhook_id = $1
+ AND d.event_id = $2
+ "#,
+ message.webhook_id,
+ message.event.event_id
+ )
+ .fetch_optional(&mut *transaction)
+ .await?;
+
+ transaction.commit().await?;
+ row.map(row_to_prepared_delivery).transpose()
+ }
+
+ #[tracing::instrument(skip(self), err)]
+ async fn cancel_delivery(&self, delivery_id: &str) -> Result<(), Self::Err> {
+ let mut transaction = self.pool.begin().await?;
+ let delivery = sqlx::query!(
+ r#"
+ SELECT webhook_id, status, attempt_count
+ FROM webhook_delivery
+ WHERE id = $1
+ FOR UPDATE
+ "#,
+ delivery_id
+ )
+ .fetch_optional(&mut *transaction)
+ .await?;
+
+ let Some(delivery) = delivery else {
+ transaction.commit().await?;
+ return Ok(());
+ };
+
+ let status = WebhookDeliveryStatus::from_str(&delivery.status)
+ .map_err(|message| decode_error("status", message))?;
+ if status.is_terminal() {
+ transaction.commit().await?;
+ return Ok(());
+ }
+
+ if status == WebhookDeliveryStatus::InProgress {
+ interrupt_current_attempt(
+ &mut transaction,
+ delivery_id,
+ &delivery.webhook_id,
+ delivery.attempt_count,
+ )
+ .await?;
+ }
+
+ sqlx::query!(
+ r#"
+ UPDATE webhook_delivery
+ SET
+ status = $2,
+ next_attempt_at = NULL,
+ updated_at = now()
+ WHERE id = $1
+ "#,
+ delivery_id,
+ WebhookDeliveryStatus::Canceled.as_str()
+ )
+ .execute(&mut *transaction)
+ .await?;
+
+ transaction.commit().await
+ }
+
+ #[tracing::instrument(skip(self), err)]
+ async fn begin_attempt(
+ &self,
+ delivery_id: &str,
+ ) -> Result<Option<WebhookDeliveryAttempt>, Self::Err> {
+ let mut transaction = self.pool.begin().await?;
+ let delivery = sqlx::query!(
+ r#"
+ SELECT
+ webhook_id,
+ status,
+ attempt_count,
+ COALESCE(next_attempt_at <= now(), true) AS "is_due!"
+ FROM webhook_delivery
+ WHERE id = $1
+ FOR UPDATE
+ "#,
+ delivery_id
+ )
+ .fetch_optional(&mut *transaction)
+ .await?;
+
+ let Some(delivery) = delivery else {
+ transaction.commit().await?;
+ return Ok(None);
+ };
+
+ let status = WebhookDeliveryStatus::from_str(&delivery.status)
+ .map_err(|message| decode_error("status", message))?;
+ if status.is_terminal() || !delivery.is_due {
+ transaction.commit().await?;
+ return Ok(None);
+ }
+
+ if status == WebhookDeliveryStatus::InProgress {
+ interrupt_current_attempt(
+ &mut transaction,
+ delivery_id,
+ &delivery.webhook_id,
+ delivery.attempt_count,
+ )
+ .await?;
+ }
+
+ let started = sqlx::query!(
+ r#"
+ UPDATE webhook_delivery
+ SET
+ status = $2,
+ attempt_count = attempt_count + 1,
+ first_attempt_at = COALESCE(first_attempt_at, now()),
+ last_attempt_at = now(),
+ next_attempt_at = NULL,
+ updated_at = now()
+ WHERE id = $1
+ RETURNING
+ attempt_count,
+ last_attempt_at AS "started_at!"
+ "#,
+ delivery_id,
+ WebhookDeliveryStatus::InProgress.as_str()
+ )
+ .fetch_one(&mut *transaction)
+ .await?;
+
+ let attempt_id = new_attempt_id();
+ sqlx::query!(
+ r#"
+ INSERT INTO webhook_delivery_attempt (
+ id,
+ webhook_delivery_id,
+ attempt_number,
+ status,
+ started_at
+ )
+ VALUES ($1, $2, $3, $4, $5)
+ "#,
+ attempt_id,
+ delivery_id,
+ started.attempt_count,
+ WebhookDeliveryAttemptStatus::InProgress.as_str(),
+ started.started_at
+ )
+ .execute(&mut *transaction)
+ .await?;
+
+ transaction.commit().await?;
+ Ok(Some(WebhookDeliveryAttempt {
+ attempt_id,
+ delivery_id: delivery_id.to_string(),
+ attempt_number: u32::try_from(started.attempt_count)
+ .map_err(|error| decode_error("attempt_count", error))?,
+ }))
+ }
+
+ #[tracing::instrument(skip(self, details), err)]
+ async fn record_success(
+ &self,
+ attempt: &WebhookDeliveryAttempt,
+ details: &WebhookHttpOutcomeDetails,
+ ) -> Result<(), Self::Err> {
+ complete_attempt(
+ &self.pool,
+ attempt,
+ details,
+ CompletionState {
+ attempt_status: WebhookDeliveryAttemptStatus::Succeeded,
+ delivery_status: WebhookDeliveryStatus::Delivered,
+ next_attempt_at: None,
+ delivered: true,
+ },
+ )
+ .await
+ }
+
+ #[tracing::instrument(skip(self, details), err)]
+ async fn record_retryable_failure(
+ &self,
+ attempt: &WebhookDeliveryAttempt,
+ details: &WebhookHttpOutcomeDetails,
+ next_attempt_at: DateTime<Utc>,
+ ) -> Result<(), Self::Err> {
+ complete_attempt(
+ &self.pool,
+ attempt,
+ details,
+ CompletionState {
+ attempt_status: WebhookDeliveryAttemptStatus::RetryableFailure,
+ delivery_status: WebhookDeliveryStatus::RetryScheduled,
+ next_attempt_at: Some(next_attempt_at),
+ delivered: false,
+ },
+ )
+ .await
+ }
+
+ #[tracing::instrument(skip(self, details), err)]
+ async fn record_permanent_failure(
+ &self,
+ attempt: &WebhookDeliveryAttempt,
+ details: &WebhookHttpOutcomeDetails,
+ ) -> Result<(), Self::Err> {
+ complete_attempt(
+ &self.pool,
+ attempt,
+ details,
+ CompletionState {
+ attempt_status: WebhookDeliveryAttemptStatus::PermanentFailure,
+ delivery_status: WebhookDeliveryStatus::PermanentlyFailed,
+ next_attempt_at: None,
+ delivered: false,
+ },
+ )
+ .await
+ }
+
+ #[tracing::instrument(skip(self, details), err)]
+ async fn record_exhaustion(
+ &self,
+ attempt: &WebhookDeliveryAttempt,
+ details: &WebhookHttpOutcomeDetails,
+ ) -> Result<(), Self::Err> {
+ complete_attempt(
+ &self.pool,
+ attempt,
+ details,
+ CompletionState {
+ attempt_status: WebhookDeliveryAttemptStatus::Exhausted,
+ delivery_status: WebhookDeliveryStatus::Exhausted,
+ next_attempt_at: None,
+ delivered: false,
+ },
+ )
+ .await
+ }
+}
diff --git a/rust/cloud-storage/webhook/src/outbound/pg_delivery_repository_test.rs b/rust/cloud-storage/webhook/src/outbound/pg_delivery_repository_test.rs
new file mode 100644
index 0000000000..f929b62add
--- /dev/null
+++ b/rust/cloud-storage/webhook/src/outbound/pg_delivery_repository_test.rs
@@ -0,0 +1,616 @@
+use super::*;
+use crate::domain::{
+ models::{
+ NormalizedWebhookEvent, WebhookDeliveryAttemptStatus, WebhookEventQueueMessage,
+ WebhookHeaders,
+ },
+ ports::WebhookDeliveryRepository,
+};
+use macro_db_migrator::MACRO_DB_MIGRATIONS;
+use serde_json::json;
+use sqlx::PgPool;
+use std::{collections::BTreeMap, time::Duration};
+
+const WEBHOOK_ID: &str = "wh_delivery_test";
+const USER_ID: &str = "macro|[email protected]";
+const ENDPOINT_URL: &str = "https://example.com/webhooks/original";
+const SIGNING_SECRET: &str = "secret-never-snapshotted";
+
+async fn insert_webhook(pool: &PgPool, deleted: bool) -> anyhow::Result<()> {
+ let deleted_at = deleted.then(Utc::now);
+ sqlx::query!(
+ r#"
+ INSERT INTO webhook (
+ id,
+ workspace_id,
+ name,
+ endpoint_url,
+ signing_secret,
+ headers,
+ filters,
+ status,
+ is_valid,
+ created_by_user_id,
+ deleted_at
+ )
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, true, $9, $10)
+ "#,
+ WEBHOOK_ID,
+ USER_ID,
+ "Delivery test",
+ ENDPOINT_URL,
+ SIGNING_SECRET,
+ json!({
+ "Authorization": "Bearer private",
+ "X-Custom": "also private",
+ }),
+ json!([{"events": ["document.created"]}]),
+ WebhookStatus::Active.as_str(),
+ USER_ID,
+ deleted_at
+ )
+ .execute(pool)
+ .await?;
+ Ok(())
+}
+
+fn message(event_id: &str, body_marker: &str) -> WebhookEventQueueMessage {
+ WebhookEventQueueMessage::new(
+ WEBHOOK_ID.to_string(),
+ NormalizedWebhookEvent {
+ event_id: event_id.to_string(),
+ schema_version: 1,
+ event_name: "document.created".to_string(),
+ entity_type: "document".to_string(),
+ entity_id: "doc_123".to_string(),
+ ordering_key: "doc_123".to_string(),
+ occurred_at: Utc::now(),
+ broker_envelope: json!({
+ "event_id": event_id,
+ "marker": body_marker,
+ "metadata": {"private": true},
+ }),
+ },
+ )
+}
+
+fn outcome_details() -> WebhookHttpOutcomeDetails {
+ WebhookHttpOutcomeDetails {
+ duration: Duration::from_millis(125),
+ response_status: Some(503),
+ response_headers_redacted: Some(BTreeMap::from([
+ ("Content-Type".to_string(), "[REDACTED]".to_string()),
+ ("X-Request-Id".to_string(), "[REDACTED]".to_string()),
+ ])),
+ response_body_preview: Some("temporarily unavailable".to_string()),
+ error_kind: Some("http_status".to_string()),
+ error_message: Some("webhook returned HTTP 503".to_string()),
+ }
+}
+
+async fn prepare(
+ repository: &PgWebhookDeliveryRepository,
+ event_id: &str,
+) -> anyhow::Result<PreparedWebhookDelivery> {
+ repository
+ .prepare_delivery(&message(event_id, "original"))
+ .await?
+ .ok_or_else(|| anyhow::anyhow!("webhook should exist"))
+}
+
+async fn begin(
+ repository: &PgWebhookDeliveryRepository,
+ delivery_id: &str,
+) -> anyhow::Result<WebhookDeliveryAttempt> {
+ repository
+ .begin_attempt(delivery_id)
+ .await?
+ .ok_or_else(|| anyhow::anyhow!("delivery should be ready"))
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn prepare_is_idempotent_and_snapshots_redacted_request_data(
+ pool: PgPool,
+) -> anyhow::Result<()> {
+ insert_webhook(&pool, false).await?;
+ let repository = PgWebhookDeliveryRepository::new(pool.clone());
+ let original_message = message("evt_idempotent", "original");
+
+ let first = repository
+ .prepare_delivery(&original_message)
+ .await?
+ .expect("prepared delivery");
+ assert!(first.delivery_id.starts_with("whd_"));
+ assert_eq!(first.status, WebhookDeliveryStatus::Queued);
+ assert_eq!(first.attempt_count, 0);
+ assert_eq!(first.webhook.signing_secret, SIGNING_SECRET);
+
+ sqlx::query!(
+ r#"
+ UPDATE webhook
+ SET
+ endpoint_url = $2,
+ headers = $3,
+ updated_at = now()
+ WHERE id = $1
+ "#,
+ WEBHOOK_ID,
+ "https://example.com/webhooks/current",
+ json!({"X-Current": "new private value"})
+ )
+ .execute(&pool)
+ .await?;
+
+ let current = repository
+ .prepare_delivery(&message("evt_idempotent", "replacement"))
+ .await?
+ .expect("existing delivery");
+ assert_eq!(current.delivery_id, first.delivery_id);
+ assert_eq!(
+ current.webhook.endpoint_url,
+ "https://example.com/webhooks/current"
+ );
+ assert_eq!(
+ current.webhook.headers,
+ WebhookHeaders::from([("X-Current".to_string(), "new private value".to_string())])
+ );
+
+ let stored = sqlx::query!(
+ r#"
+ SELECT
+ request_url,
+ request_headers_redacted,
+ request_body,
+ event,
+ event_schema_version,
+ event_entity_type,
+ event_entity_id,
+ event_ordering_key
+ FROM webhook_delivery
+ WHERE webhook_id = $1 AND event_id = $2
+ "#,
+ WEBHOOK_ID,
+ "evt_idempotent"
+ )
+ .fetch_one(&pool)
+ .await?;
+ let delivery_count = sqlx::query_scalar!(
+ "SELECT COUNT(*) FROM webhook_delivery WHERE webhook_id = $1 AND event_id = $2",
+ WEBHOOK_ID,
+ "evt_idempotent"
+ )
+ .fetch_one(&pool)
+ .await?;
+
+ assert_eq!(delivery_count, Some(1));
+ assert_eq!(stored.request_url, ENDPOINT_URL);
+ assert_eq!(
+ stored.request_headers_redacted,
+ json!({
+ "Authorization": REDACTED_HEADER_VALUE,
+ "X-Custom": REDACTED_HEADER_VALUE,
+ })
+ );
+ assert_eq!(stored.request_body, original_message.event.broker_envelope);
+ assert_eq!(stored.event, "document.created");
+ assert_eq!(stored.event_schema_version, 1);
+ assert_eq!(stored.event_entity_type, "document");
+ assert_eq!(stored.event_entity_id, "doc_123");
+ assert_eq!(stored.event_ordering_key, "doc_123");
+ Ok(())
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn prepare_returns_none_for_a_missing_webhook(pool: PgPool) -> anyhow::Result<()> {
+ let repository = PgWebhookDeliveryRepository::new(pool.clone());
+
+ assert!(
+ repository
+ .prepare_delivery(&message("evt_missing", "original"))
+ .await?
+ .is_none()
+ );
+ assert_eq!(
+ sqlx::query_scalar!("SELECT COUNT(*) FROM webhook_delivery")
+ .fetch_one(&pool)
+ .await?,
+ Some(0)
+ );
+ Ok(())
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn retry_scheduling_blocks_early_attempts_and_attempt_numbers_increase(
+ pool: PgPool,
+) -> anyhow::Result<()> {
+ insert_webhook(&pool, false).await?;
+ let repository = PgWebhookDeliveryRepository::new(pool.clone());
+ let prepared = prepare(&repository, "evt_retry").await?;
+ let first_attempt = begin(&repository, &prepared.delivery_id).await?;
+ assert!(first_attempt.attempt_id.starts_with("wha_"));
+ assert_eq!(first_attempt.attempt_number, 1);
+
+ let next_attempt_at = Utc::now() + chrono::Duration::hours(1);
+ repository
+ .record_retryable_failure(&first_attempt, &outcome_details(), next_attempt_at)
+ .await?;
+
+ let scheduled = repository
+ .prepare_delivery(&message("evt_retry", "duplicate"))
+ .await?
+ .expect("scheduled delivery");
+ assert_eq!(scheduled.status, WebhookDeliveryStatus::RetryScheduled);
+ assert_eq!(scheduled.attempt_count, 1);
+ assert!(
+ scheduled
+ .next_attempt_at
+ .is_some_and(|due| due > Utc::now())
+ );
+ assert!(
+ repository
+ .begin_attempt(&prepared.delivery_id)
+ .await?
+ .is_none()
+ );
+
+ sqlx::query!(
+ r#"
+ UPDATE webhook_delivery
+ SET next_attempt_at = now() - INTERVAL '1 second'
+ WHERE id = $1
+ "#,
+ prepared.delivery_id
+ )
+ .execute(&pool)
+ .await?;
+
+ let second_attempt = begin(&repository, &prepared.delivery_id).await?;
+ assert_eq!(second_attempt.attempt_number, 2);
+ assert_ne!(second_attempt.attempt_id, first_attempt.attempt_id);
+
+ let state = sqlx::query!(
+ r#"
+ SELECT attempt_count, first_attempt_at, last_attempt_at, next_attempt_at
+ FROM webhook_delivery
+ WHERE id = $1
+ "#,
+ prepared.delivery_id
+ )
+ .fetch_one(&pool)
+ .await?;
+ let attempts = sqlx::query!(
+ r#"
+ SELECT attempt_number, status
+ FROM webhook_delivery_attempt
+ WHERE webhook_delivery_id = $1
+ ORDER BY attempt_number
+ "#,
+ prepared.delivery_id
+ )
+ .fetch_all(&pool)
+ .await?;
+ let aggregate = sqlx::query!(
+ "SELECT last_failure_at FROM webhook WHERE id = $1",
+ WEBHOOK_ID
+ )
+ .fetch_one(&pool)
+ .await?;
+
+ assert_eq!(state.attempt_count, 2);
+ assert!(state.first_attempt_at.is_some());
+ assert!(state.last_attempt_at >= state.first_attempt_at);
+ assert!(state.next_attempt_at.is_none());
+ assert_eq!(attempts.len(), 2);
+ assert_eq!(attempts[0].attempt_number, 1);
+ assert_eq!(
+ attempts[0].status,
+ WebhookDeliveryAttemptStatus::RetryableFailure.as_str()
+ );
+ assert_eq!(attempts[1].attempt_number, 2);
+ assert_eq!(
+ attempts[1].status,
+ WebhookDeliveryAttemptStatus::InProgress.as_str()
+ );
+ assert!(aggregate.last_failure_at.is_some());
+ Ok(())
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn soft_deleted_webhook_delivery_can_be_canceled_without_an_attempt(
+ pool: PgPool,
+) -> anyhow::Result<()> {
+ insert_webhook(&pool, true).await?;
+ let repository = PgWebhookDeliveryRepository::new(pool.clone());
+ let prepared = prepare(&repository, "evt_canceled").await?;
+ assert!(prepared.webhook.deleted_at.is_some());
+
+ repository.cancel_delivery(&prepared.delivery_id).await?;
+ repository.cancel_delivery(&prepared.delivery_id).await?;
+
+ let delivery = sqlx::query!(
+ "SELECT status, attempt_count FROM webhook_delivery WHERE id = $1",
+ prepared.delivery_id
+ )
+ .fetch_one(&pool)
+ .await?;
+ let attempt_count = sqlx::query_scalar!(
+ "SELECT COUNT(*) FROM webhook_delivery_attempt WHERE webhook_delivery_id = $1",
+ prepared.delivery_id
+ )
+ .fetch_one(&pool)
+ .await?;
+ let aggregate = sqlx::query!(
+ "SELECT last_success_at, last_failure_at FROM webhook WHERE id = $1",
+ WEBHOOK_ID
+ )
+ .fetch_one(&pool)
+ .await?;
+
+ assert_eq!(delivery.status, WebhookDeliveryStatus::Canceled.as_str());
+ assert_eq!(delivery.attempt_count, 0);
+ assert_eq!(attempt_count, Some(0));
+ assert!(aggregate.last_success_at.is_none());
+ assert!(aggregate.last_failure_at.is_none());
+ assert!(
+ repository
+ .begin_attempt(&prepared.delivery_id)
+ .await?
+ .is_none()
+ );
+ Ok(())
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn success_is_terminal_and_updates_success_aggregate_atomically(
+ pool: PgPool,
+) -> anyhow::Result<()> {
+ insert_webhook(&pool, false).await?;
+ let repository = PgWebhookDeliveryRepository::new(pool.clone());
+ let prepared = prepare(&repository, "evt_success").await?;
+ let attempt = begin(&repository, &prepared.delivery_id).await?;
+ let mut details = outcome_details();
+ details.response_status = Some(204);
+ details.error_kind = None;
+ details.error_message = None;
+
+ repository.record_success(&attempt, &details).await?;
+
+ let duplicate = repository
+ .prepare_delivery(&message("evt_success", "redelivery"))
+ .await?
+ .expect("terminal delivery");
+ assert_eq!(duplicate.delivery_id, prepared.delivery_id);
+ assert_eq!(duplicate.status, WebhookDeliveryStatus::Delivered);
+ assert!(
+ repository
+ .begin_attempt(&prepared.delivery_id)
+ .await?
+ .is_none()
+ );
+
+ let state = sqlx::query!(
+ r#"
+ SELECT
+ d.status,
+ d.delivered_at,
+ d.attempt_count,
+ a.status AS attempt_status,
+ a.completed_at,
+ w.last_success_at,
+ w.last_failure_at
+ FROM webhook_delivery d
+ JOIN webhook_delivery_attempt a ON a.webhook_delivery_id = d.id
+ JOIN webhook w ON w.id = d.webhook_id
+ WHERE d.id = $1
+ "#,
+ prepared.delivery_id
+ )
+ .fetch_one(&pool)
+ .await?;
+
+ assert_eq!(state.status, WebhookDeliveryStatus::Delivered.as_str());
+ assert_eq!(state.attempt_count, 1);
+ assert_eq!(
+ state.attempt_status,
+ WebhookDeliveryAttemptStatus::Succeeded.as_str()
+ );
+ assert!(state.delivered_at.is_some());
+ assert_eq!(state.delivered_at, state.completed_at);
+ assert_eq!(state.delivered_at, state.last_success_at);
+ assert!(state.last_failure_at.is_none());
+ Ok(())
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn permanent_failure_is_terminal_and_updates_failure_aggregate(
+ pool: PgPool,
+) -> anyhow::Result<()> {
+ insert_webhook(&pool, false).await?;
+ let repository = PgWebhookDeliveryRepository::new(pool.clone());
+ let prepared = prepare(&repository, "evt_permanent").await?;
+ let attempt = begin(&repository, &prepared.delivery_id).await?;
+
+ repository
+ .record_permanent_failure(&attempt, &outcome_details())
+ .await?;
+
+ let state = sqlx::query!(
+ r#"
+ SELECT
+ d.status,
+ d.delivered_at,
+ a.status AS attempt_status,
+ a.completed_at,
+ a.duration_ms,
+ a.response_status,
+ a.response_headers_redacted,
+ a.response_body_preview,
+ a.error_kind,
+ a.error_message,
+ w.last_success_at,
+ w.last_failure_at
+ FROM webhook_delivery d
+ JOIN webhook_delivery_attempt a ON a.webhook_delivery_id = d.id
+ JOIN webhook w ON w.id = d.webhook_id
+ WHERE d.id = $1
+ "#,
+ prepared.delivery_id
+ )
+ .fetch_one(&pool)
+ .await?;
+
+ assert_eq!(
+ state.status,
+ WebhookDeliveryStatus::PermanentlyFailed.as_str()
+ );
+ assert_eq!(
+ state.attempt_status,
+ WebhookDeliveryAttemptStatus::PermanentFailure.as_str()
+ );
+ assert!(state.delivered_at.is_none());
+ assert_eq!(state.duration_ms, Some(125));
+ assert_eq!(state.response_status, Some(503));
+ assert_eq!(
+ state.response_headers_redacted,
+ Some(json!({
+ "Content-Type": "[REDACTED]",
+ "X-Request-Id": "[REDACTED]",
+ }))
+ );
+ assert_eq!(
+ state.response_body_preview.as_deref(),
+ Some("temporarily unavailable")
+ );
+ assert_eq!(state.error_kind.as_deref(), Some("http_status"));
+ assert_eq!(
+ state.error_message.as_deref(),
+ Some("webhook returned HTTP 503")
+ );
+ assert_eq!(state.completed_at, state.last_failure_at);
+ assert!(state.last_success_at.is_none());
+ assert!(
+ repository
+ .begin_attempt(&prepared.delivery_id)
+ .await?
+ .is_none()
+ );
+ Ok(())
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn exhaustion_is_terminal_and_updates_failure_aggregate(pool: PgPool) -> anyhow::Result<()> {
+ insert_webhook(&pool, false).await?;
+ let repository = PgWebhookDeliveryRepository::new(pool.clone());
+ let prepared = prepare(&repository, "evt_exhausted").await?;
+ let attempt = begin(&repository, &prepared.delivery_id).await?;
+
+ repository
+ .record_exhaustion(&attempt, &outcome_details())
+ .await?;
+
+ let state = sqlx::query!(
+ r#"
+ SELECT
+ d.status,
+ d.next_attempt_at,
+ a.status AS attempt_status,
+ a.completed_at,
+ w.last_failure_at
+ FROM webhook_delivery d
+ JOIN webhook_delivery_attempt a ON a.webhook_delivery_id = d.id
+ JOIN webhook w ON w.id = d.webhook_id
+ WHERE d.id = $1
+ "#,
+ prepared.delivery_id
+ )
+ .fetch_one(&pool)
+ .await?;
+
+ assert_eq!(state.status, WebhookDeliveryStatus::Exhausted.as_str());
+ assert_eq!(
+ state.attempt_status,
+ WebhookDeliveryAttemptStatus::Exhausted.as_str()
+ );
+ assert!(state.next_attempt_at.is_none());
+ assert_eq!(state.completed_at, state.last_failure_at);
+ assert!(
+ repository
+ .begin_attempt(&prepared.delivery_id)
+ .await?
+ .is_none()
+ );
+ Ok(())
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn interrupted_attempt_is_recorded_and_recovered_monotonically(
+ pool: PgPool,
+) -> anyhow::Result<()> {
+ insert_webhook(&pool, false).await?;
+ let repository = PgWebhookDeliveryRepository::new(pool.clone());
+ let prepared = prepare(&repository, "evt_interrupted").await?;
+ let interrupted_attempt = begin(&repository, &prepared.delivery_id).await?;
+
+ let recovered_attempt = begin(&repository, &prepared.delivery_id).await?;
+ assert_eq!(interrupted_attempt.attempt_number, 1);
+ assert_eq!(recovered_attempt.attempt_number, 2);
+
+ let attempts = sqlx::query!(
+ r#"
+ SELECT attempt_number, status, completed_at
+ FROM webhook_delivery_attempt
+ WHERE webhook_delivery_id = $1
+ ORDER BY attempt_number
+ "#,
+ prepared.delivery_id
+ )
+ .fetch_all(&pool)
+ .await?;
+ let aggregate = sqlx::query!(
+ "SELECT last_failure_at FROM webhook WHERE id = $1",
+ WEBHOOK_ID
+ )
+ .fetch_one(&pool)
+ .await?;
+
+ assert_eq!(attempts.len(), 2);
+ assert_eq!(
+ attempts[0].status,
+ WebhookDeliveryAttemptStatus::Interrupted.as_str()
+ );
+ assert!(attempts[0].completed_at.is_some());
+ assert_eq!(attempts[0].completed_at, aggregate.last_failure_at);
+ assert_eq!(
+ attempts[1].status,
+ WebhookDeliveryAttemptStatus::InProgress.as_str()
+ );
+ assert!(attempts[1].completed_at.is_none());
+
+ let stale_result = repository
+ .record_success(&interrupted_attempt, &outcome_details())
+ .await;
+ assert!(matches!(stale_result, Err(sqlx::Error::RowNotFound)));
+
+ let before_completion = sqlx::query!(
+ "SELECT status, attempt_count FROM webhook_delivery WHERE id = $1",
+ prepared.delivery_id
+ )
+ .fetch_one(&pool)
+ .await?;
+ assert_eq!(
+ before_completion.status,
+ WebhookDeliveryStatus::InProgress.as_str()
+ );
+ assert_eq!(before_completion.attempt_count, 2);
+
+ repository
+ .record_success(&recovered_attempt, &outcome_details())
+ .await?;
+ let recovered = repository
+ .prepare_delivery(&message("evt_interrupted", "redelivery"))
+ .await?
+ .expect("recovered delivery");
+ assert_eq!(recovered.status, WebhookDeliveryStatus::Delivered);
+ assert_eq!(recovered.attempt_count, 2);
+ Ok(())
+}
diff --git a/rust/cloud-storage/webhook/src/outbound/pg_repository.rs b/rust/cloud-storage/webhook/src/outbound/pg_repository.rs
index 9e8e333504..0a8de938ae 100644
--- a/rust/cloud-storage/webhook/src/outbound/pg_repository.rs
+++ b/rust/cloud-storage/webhook/src/outbound/pg_repository.rs
@@ -8,7 +8,7 @@ use crate::domain::{
CreateWebhookRequest, PatchWebhookRequest, Webhook, WebhookFilters, WebhookHeaders,
WebhookId, WebhookStatus,
},
- ports::WebhookRepo,
+ ports::{WebhookRepo, WebhookWorkspaceResolver},
};
use macro_user_id::user_id::MacroUserIdStr;
use serde_json::Value;
@@ -115,6 +115,44 @@ async fn fetch_webhook(pool: &PgPool, webhook_id: &str) -> Result<Option<Webhook
row.map(row_to_webhook).transpose()
}
+impl WebhookWorkspaceResolver for PgRepository {
+ type Err = sqlx::Error;
+
+ #[tracing::instrument(skip(self, people), err)]
+ async fn resolve_workspace_ids(
+ &self,
+ people: Vec<MacroUserIdStr<'static>>,
+ ) -> Result<Vec<String>, Self::Err> {
+ let mut workspace_ids = people
+ .iter()
+ .map(|person| person.as_ref().to_string())
+ .collect::<Vec<_>>();
+ workspace_ids.sort_unstable();
+ workspace_ids.dedup();
+
+ if workspace_ids.is_empty() {
+ return Ok(workspace_ids);
+ }
+
+ let team_workspace_ids = sqlx::query_scalar!(
+ r#"
+ SELECT DISTINCT team_id::text AS "team_id!"
+ FROM team_user
+ WHERE user_id = ANY($1)
+ ORDER BY team_id::text
+ "#,
+ &workspace_ids
+ )
+ .fetch_all(&self.pool)
+ .await?;
+
+ workspace_ids.extend(team_workspace_ids);
+ workspace_ids.sort_unstable();
+ workspace_ids.dedup();
+ Ok(workspace_ids)
+ }
+}
+
impl WebhookRepo for PgRepository {
type Err = sqlx::Error;
diff --git a/rust/cloud-storage/webhook/src/outbound/pg_repository_test.rs b/rust/cloud-storage/webhook/src/outbound/pg_repository_test.rs
index d9bba1d9b6..295450216a 100644
--- a/rust/cloud-storage/webhook/src/outbound/pg_repository_test.rs
+++ b/rust/cloud-storage/webhook/src/outbound/pg_repository_test.rs
@@ -1,7 +1,7 @@
use super::*;
use crate::domain::{
models::{WebhookFilter, WebhookStatus},
- ports::WebhookRepo,
+ ports::{WebhookRepo, WebhookWorkspaceResolver},
};
use macro_db_migrator::MACRO_DB_MIGRATIONS;
use macro_user_id::user_id::MacroUserIdStr;
@@ -9,11 +9,21 @@ use serde_json::json;
use sqlx::{PgPool, types::Uuid};
const USER_ID: &str = "macro|[email protected]";
+const SECOND_USER_ID: &str = "macro|[email protected]";
const TEAM_ID: &str = "11111111-1111-1111-1111-111111111111";
+const UNRELATED_TEAM_ID: &str = "22222222-2222-2222-2222-222222222222";
const OTHER_WORKSPACE_ID: &str = "workspace_other";
+fn macro_user_id(user_id: &str) -> MacroUserIdStr<'static> {
+ MacroUserIdStr::try_from(user_id.to_string()).expect("valid macro user id")
+}
+
fn user_id() -> MacroUserIdStr<'static> {
- MacroUserIdStr::try_from(USER_ID.to_string()).expect("valid macro user id")
+ macro_user_id(USER_ID)
+}
+
+fn second_user_id() -> MacroUserIdStr<'static> {
+ macro_user_id(SECOND_USER_ID)
}
fn create_request() -> CreateWebhookRequest {
@@ -29,7 +39,7 @@ fn create_request() -> CreateWebhookRequest {
}
}
-async fn insert_user(pool: &PgPool) -> anyhow::Result<()> {
+async fn insert_user_with_id(pool: &PgPool, user_id: &str, email: &str) -> anyhow::Result<()> {
let macro_user_id = macro_uuid::generate_uuid_v7();
let stripe_customer_id = format!("stripe_{macro_user_id}");
sqlx::query!(
@@ -39,8 +49,8 @@ async fn insert_user(pool: &PgPool) -> anyhow::Result<()> {
ON CONFLICT (id) DO NOTHING
"#,
macro_user_id,
+ email,
+ email,
stripe_customer_id
)
.execute(pool)
@@ -52,8 +62,8 @@ async fn insert_user(pool: &PgPool) -> anyhow::Result<()> {
VALUES ($1, $2, $3)
ON CONFLICT (id) DO NOTHING
"#,
- USER_ID,
+ user_id,
+ email,
macro_user_id
)
.execute(pool)
@@ -62,6 +72,33 @@ async fn insert_user(pool: &PgPool) -> anyhow::Result<()> {
Ok(())
}
+async fn insert_user(pool: &PgPool) -> anyhow::Result<()> {
+ insert_user_with_id(pool, USER_ID, "[email protected]").await
+}
+
+async fn insert_team(pool: &PgPool, team_id: Uuid, owner_id: &str) -> anyhow::Result<()> {
+ sqlx::query!(
+ r#"INSERT INTO team (id, owner_id, name) VALUES ($1, $2, $3)"#,
+ team_id,
+ owner_id,
+ team_id.to_string(),
+ )
+ .execute(pool)
+ .await?;
+ Ok(())
+}
+
+async fn insert_team_member(pool: &PgPool, user_id: &str, team_id: Uuid) -> anyhow::Result<()> {
+ sqlx::query!(
+ r#"INSERT INTO team_user (user_id, team_id, team_role) VALUES ($1, $2, 'member')"#,
+ user_id,
+ team_id
+ )
+ .execute(pool)
+ .await?;
+ Ok(())
+}
+
async fn create_webhook(repo: &PgRepository) -> Webhook {
repo.create_webhook(
user_id(),
@@ -363,6 +400,92 @@ async fn list_matching_event_excludes_ineligible_webhooks(pool: PgPool) -> anyho
Ok(())
}
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn resolved_workspaces_match_personal_and_related_team_webhooks(
+ pool: PgPool,
+) -> anyhow::Result<()> {
+ insert_user(&pool).await?;
+ insert_user_with_id(&pool, SECOND_USER_ID, "[email protected]").await?;
+
+ let team_id = Uuid::parse_str(TEAM_ID)?;
+ let unrelated_team_id = Uuid::parse_str(UNRELATED_TEAM_ID)?;
+ insert_team(&pool, team_id, USER_ID).await?;
+ insert_team(&pool, unrelated_team_id, SECOND_USER_ID).await?;
+ insert_team_member(&pool, USER_ID, team_id).await?;
+ insert_team_member(&pool, SECOND_USER_ID, team_id).await?;
+
+ let matching_filter = vec![webhook_filter(&["document.created"], None)];
+ insert_webhook_for_matching(
+ &pool,
+ "wh_personal",
+ USER_ID,
+ matching_filter.clone(),
+ WebhookStatus::Active,
+ true,
+ false,
+ )
+ .await?;
+ insert_webhook_for_matching(
+ &pool,
+ "wh_team",
+ TEAM_ID,
+ matching_filter.clone(),
+ WebhookStatus::Active,
+ true,
+ false,
+ )
+ .await?;
+ insert_webhook_for_matching(
+ &pool,
+ "wh_unrelated_team",
+ UNRELATED_TEAM_ID,
+ matching_filter,
+ WebhookStatus::Active,
+ true,
+ false,
+ )
+ .await?;
+
+ let repo = PgRepository::new(pool);
+ let workspace_ids = repo
+ .resolve_workspace_ids(vec![
+ second_user_id(),
+ user_id(),
+ second_user_id(),
+ user_id(),
+ ])
+ .await?;
+
+ assert_eq!(
+ workspace_ids,
+ vec![
+ TEAM_ID.to_string(),
+ USER_ID.to_string(),
+ SECOND_USER_ID.to_string(),
+ ]
+ );
+
+ let webhooks = repo
+ .list_active_webhooks_matching_event(
+ workspace_ids,
+ "document.created".to_string(),
+ "doc_1".to_string(),
+ )
+ .await?;
+
+ assert_eq!(webhook_ids(&webhooks), vec!["wh_personal", "wh_team"]);
+ Ok(())
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn resolve_workspace_ids_returns_empty_for_no_people(pool: PgPool) -> anyhow::Result<()> {
+ let repo = PgRepository::new(pool.clone());
+ pool.close().await;
+
+ assert!(repo.resolve_workspace_ids(Vec::new()).await?.is_empty());
+ Ok(())
+}
+
#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
async fn patch_updates_endpoint_and_resets_validity(pool: PgPool) -> anyhow::Result<()> {
insert_user(&pool).await?;
@@ -431,21 +554,8 @@ async fn get_webhook_excludes_deleted_rows(pool: PgPool) -> anyhow::Result<()> {
async fn get_user_team_workspace_id_returns_team_membership(pool: PgPool) -> anyhow::Result<()> {
insert_user(&pool).await?;
let team_id = Uuid::parse_str(TEAM_ID)?;
- sqlx::query!(
- r#"INSERT INTO team (id, owner_id, name) VALUES ($1, $2, $3)"#,
- team_id,
- USER_ID,
- team_id.to_string(),
- )
- .execute(&pool)
- .await?;
- sqlx::query!(
- r#"INSERT INTO team_user (user_id, team_id, team_role) VALUES ($1, $2, 'member')"#,
- USER_ID,
- team_id
- )
- .execute(&pool)
- .await?;
+ insert_team(&pool, team_id, USER_ID).await?;
+ insert_team_member(&pool, USER_ID, team_id).await?;
let repo = PgRepository::new(pool);
assert_eq!(
diff --git a/rust/cloud-storage/webhook/src/outbound/sqs_queue.rs b/rust/cloud-storage/webhook/src/outbound/sqs_queue.rs
new file mode 100644
index 0000000000..b29888b53a
--- /dev/null
+++ b/rust/cloud-storage/webhook/src/outbound/sqs_queue.rs
@@ -0,0 +1,148 @@
+//! SQS-backed webhook event queue.
+
+#[cfg(test)]
+#[path = "sqs_queue_test.rs"]
+mod sqs_queue_test;
+
+use std::{sync::Arc, time::Duration};
+
+use anyhow::{Context, ensure};
+use sqs_client::{ReceivedMessage, SQS};
+
+use crate::domain::{
+ models::{RawWebhookEventQueueMessage, WebhookEventQueueMessage},
+ ports::{WebhookEventEnqueuer, WebhookEventQueue},
+};
+
+const MAX_VISIBILITY_TIMEOUT_SECONDS: u64 = 12 * 60 * 60;
+
+/// SQS-backed implementation of webhook enqueueing and worker queue operations.
+#[derive(Clone)]
+pub struct SqsWebhookQueue {
+ sqs: Arc<SQS>,
+ queue_url: String,
+ max_messages: i32,
+ wait_time_seconds: i32,
+}
+
+impl SqsWebhookQueue {
+ /// Create a webhook queue adapter pointing at `queue_url`.
+ pub fn new(
+ sqs: Arc<SQS>,
+ queue_url: String,
+ max_messages: i32,
+ wait_time_seconds: i32,
+ ) -> Self {
+ Self {
+ sqs,
+ queue_url,
+ max_messages,
+ wait_time_seconds,
+ }
+ }
+}
+
+impl WebhookEventEnqueuer for SqsWebhookQueue {
+ type Err = anyhow::Error;
+
+ #[tracing::instrument(
+ err,
+ skip(self, message),
+ fields(
+ webhook_id = %message.webhook_id,
+ event_id = %message.event.event_id,
+ )
+ )]
+ async fn enqueue(&self, message: WebhookEventQueueMessage) -> Result<(), Self::Err> {
+ let prepared = prepare_webhook_message(&message)?;
+
+ self.sqs
+ .send_fifo_message(
+ &self.queue_url,
+ prepared.body,
+ prepared.group_id,
+ prepared.deduplication_id,
+ )
+ .await?;
+
+ Ok(())
+ }
+}
+
+impl WebhookEventQueue for SqsWebhookQueue {
+ type Err = anyhow::Error;
+
+ #[tracing::instrument(err, skip(self))]
+ async fn receive_messages(&self) -> Result<Vec<RawWebhookEventQueueMessage>, Self::Err> {
+ let messages = self
+ .sqs
+ .receive_messages(&self.queue_url, self.max_messages, self.wait_time_seconds)
+ .await?;
+
+ Ok(messages.into_iter().map(raw_webhook_message).collect())
+ }
+
+ #[tracing::instrument(err, skip(self, receipt_handle))]
+ async fn delete_message(&self, receipt_handle: &str) -> Result<(), Self::Err> {
+ self.sqs
+ .delete_message(&self.queue_url, receipt_handle)
+ .await?;
+
+ Ok(())
+ }
+
+ #[tracing::instrument(
+ err,
+ skip(self, receipt_handle),
+ fields(delay_seconds = delay.as_secs())
+ )]
+ async fn delay_message(&self, receipt_handle: &str, delay: Duration) -> Result<(), Self::Err> {
+ let visibility_timeout = visibility_timeout_seconds(delay)?;
+
+ self.sqs
+ .change_message_visibility(&self.queue_url, receipt_handle, visibility_timeout)
+ .await?;
+
+ Ok(())
+ }
+}
+
+#[derive(Debug, PartialEq, Eq)]
+struct PreparedWebhookMessage {
+ body: String,
+ group_id: String,
+ deduplication_id: String,
+}
+
+fn prepare_webhook_message(
+ message: &WebhookEventQueueMessage,
+) -> anyhow::Result<PreparedWebhookMessage> {
+ let group_id = message.webhook_id.clone();
+ let deduplication_id = format!("{}:{}", message.webhook_id, message.event.event_id);
+ let body =
+ serde_json::to_string(message).context("unable to serialize webhook queue message")?;
+
+ Ok(PreparedWebhookMessage {
+ body,
+ group_id,
+ deduplication_id,
+ })
+}
+
+fn raw_webhook_message(message: ReceivedMessage) -> RawWebhookEventQueueMessage {
+ RawWebhookEventQueueMessage {
+ message_id: message.message_id,
+ body: message.body,
+ receipt_handle: message.receipt_handle,
+ }
+}
+
+fn visibility_timeout_seconds(delay: Duration) -> anyhow::Result<i32> {
+ let seconds = delay.as_secs();
+ ensure!(
+ seconds <= MAX_VISIBILITY_TIMEOUT_SECONDS,
+ "webhook message visibility delay cannot exceed {MAX_VISIBILITY_TIMEOUT_SECONDS} seconds"
+ );
+
+ Ok(seconds as i32)
+}
diff --git a/rust/cloud-storage/webhook/src/outbound/sqs_queue_test.rs b/rust/cloud-storage/webhook/src/outbound/sqs_queue_test.rs
new file mode 100644
index 0000000000..6a55411bee
--- /dev/null
+++ b/rust/cloud-storage/webhook/src/outbound/sqs_queue_test.rs
@@ -0,0 +1,74 @@
+use super::*;
+use crate::domain::models::WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION;
+use serde_json::json;
+
+fn sample_message() -> WebhookEventQueueMessage {
+ serde_json::from_value(json!({
+ "version": WEBHOOK_EVENT_QUEUE_MESSAGE_VERSION,
+ "webhook_id": "wh_123",
+ "event": {
+ "event_id": "event_456",
+ "schema_version": 1,
+ "event_name": "document.created",
+ "entity_type": "document",
+ "entity_id": "doc_789",
+ "ordering_key": "doc_789",
+ "occurred_at": "2026-07-10T12:00:00Z",
+ "broker_envelope": {
+ "event_id": "event_456",
+ "payload": { "document_id": "doc_789" }
+ }
+ }
+ }))
+ .unwrap()
+}
+
+#[test]
+fn webhook_message_is_serialized_as_the_typed_queue_contract() {
+ let message = sample_message();
+ let prepared = prepare_webhook_message(&message).unwrap();
+ let deserialized: WebhookEventQueueMessage = serde_json::from_str(&prepared.body).unwrap();
+
+ assert_eq!(deserialized, message);
+}
+
+#[test]
+fn webhook_message_uses_webhook_id_as_fifo_group_id() {
+ let prepared = prepare_webhook_message(&sample_message()).unwrap();
+
+ assert_eq!(prepared.group_id, "wh_123");
+}
+
+#[test]
+fn webhook_message_combines_webhook_and_event_ids_for_fifo_deduplication() {
+ let prepared = prepare_webhook_message(&sample_message()).unwrap();
+
+ assert_eq!(prepared.deduplication_id, "wh_123:event_456");
+}
+
+#[test]
+fn raw_webhook_message_retains_transport_fields() {
+ let raw = raw_webhook_message(ReceivedMessage {
+ message_id: Some("message-id".to_string()),
+ body: Some("message-body".to_string()),
+ receipt_handle: Some("receipt-handle".to_string()),
+ });
+
+ assert_eq!(raw.message_id.as_deref(), Some("message-id"));
+ assert_eq!(raw.body.as_deref(), Some("message-body"));
+ assert_eq!(raw.receipt_handle.as_deref(), Some("receipt-handle"));
+}
+
+#[test]
+fn visibility_timeout_rejects_values_above_the_sqs_limit() {
+ assert_eq!(
+ visibility_timeout_seconds(Duration::from_secs(43_200)).unwrap(),
+ 43_200,
+ );
+ assert_eq!(
+ visibility_timeout_seconds(Duration::from_secs(43_201))
+ .unwrap_err()
+ .to_string(),
+ "webhook message visibility delay cannot exceed 43200 seconds",
+ );
+}