Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

feat(webhook): keep failure payloads and add delivery-failure redrive

Yudhi Armyndharisf79e474f5ef7
diff --git a/.env.example b/.env.example
index 7d7e0bb53..19b856b7a 100644
--- a/.env.example
+++ b/.env.example
@@ -473,6 +473,12 @@ WEBHOOK_RETRY_DELAY=5000 # Base delay between retries in ms (doubles
# (404 when nothing was stored).
# Days to keep webhook delivery-failure records before pruning them (default 90; set <= 0 to disable).
# WEBHOOK_FAILURE_RETENTION_DAYS=90
+# Hours a terminal delivery-failure row keeps the event data it was built from, so
+# POST /api/webhooks/delivery-failures/redrive can replay the lost event (default 0 = keep nothing,
+# no redrive). The payload can carry a whole message body; once the window passes it is cleared and
+# the row stays. With the queue enabled each job also carries this copy in Redis, up to
+# WEBHOOK_MEDIA_INLINE_MAX_BYTES more per job.
+# WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS=0
# Block outbound WEBHOOK deliveries to internal/reserved addresses (SSRF
# protection). ON by default; set to "false" only on closed networks. When on,
# webhook URLs resolving to loopback/private/link-local/metadata ranges are
diff --git a/CHANGELOG.md b/CHANGELOG.md
index fff3ec386..9922b7e4a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
+### Added
+
+- `POST /api/webhooks/delivery-failures/redrive` (ADMIN) replays recorded webhook deliveries that still hold their event data, in batches of up to 500, oldest first, with the stored idempotency key so a receiver can dedup a replay; `WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS` sets how long a terminal failure row keeps that data (default `0`: not kept, nothing to replay), and `GET /api/webhooks/delivery-failures` marks each row `replayable`. All five SDKs gain the call in their next release.
+
## [0.24.0] - 2026-10-03
### Added
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index bbae0c367..f46cb97c1 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -207,6 +207,7 @@ services:
- WEBHOOK_MAX_PAYLOAD_BYTES=${WEBHOOK_MAX_PAYLOAD_BYTES:-}
- WEBHOOK_MAX_PER_SESSION=${WEBHOOK_MAX_PER_SESSION:-}
- WEBHOOK_FAILURE_RETENTION_DAYS=${WEBHOOK_FAILURE_RETENTION_DAYS:-}
+ - WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS=${WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS:-}
- WEBHOOK_WORKER_CONCURRENCY=${WEBHOOK_WORKER_CONCURRENCY:-}
- WEBHOOK_DEGRADED_SESSION_CONCURRENCY=${WEBHOOK_DEGRADED_SESSION_CONCURRENCY:-}
# Integration Fabric ingress: fairness, retries, retention, reconciler, workers.
diff --git a/docker-compose.yml b/docker-compose.yml
index 43a56152d..34b4ee732 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -303,6 +303,7 @@ services:
- WEBHOOK_MAX_PAYLOAD_BYTES=${WEBHOOK_MAX_PAYLOAD_BYTES:-}
- WEBHOOK_MAX_PER_SESSION=${WEBHOOK_MAX_PER_SESSION:-}
- WEBHOOK_FAILURE_RETENTION_DAYS=${WEBHOOK_FAILURE_RETENTION_DAYS:-}
+ - WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS=${WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS:-}
- WEBHOOK_WORKER_CONCURRENCY=${WEBHOOK_WORKER_CONCURRENCY:-}
- WEBHOOK_DEGRADED_SESSION_CONCURRENCY=${WEBHOOK_DEGRADED_SESSION_CONCURRENCY:-}
# Integration Fabric ingress: per-instance fairness, retries, retention, reconciler, workers.
diff --git a/docs/04-security-design.md b/docs/04-security-design.md
index fab252291..3bf6c4785 100644
--- a/docs/04-security-design.md
+++ b/docs/04-security-design.md
@@ -426,6 +426,15 @@ with the payload limit at least 4/3 of the inline limit plus room for the envelo
gateway applies only the inline limit. Media the engine downloaded stays retrievable from
`GET /api/sessions/:sessionId/messages/:chatId/:messageId/media` (`404` when nothing was stored).
+A `webhook_delivery_failures` row holds no event data by default. With
+`WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS` > 0 a terminal row also keeps the pre-hook event data
+(already media-shed, so at most about `WEBHOOK_MEDIA_INLINE_MAX_BYTES` of media) so an ADMIN can
+replay it with `POST /api/webhooks/delivery-failures/redrive`. That is message content at rest in
+the data database: it is cleared once the window passes (checked at startup and then hourly; setting
+the knob back to `0` clears every stored copy on the next start), it is never returned by the list
+route, and `GET /api/infra/export-data` drops it from backups. With the queue enabled each job also
+carries the copy in Redis for the queue's retention window.
+
## 4.9 Security Headers
### Helmet Configuration
diff --git a/docs/05-database-design.md b/docs/05-database-design.md
index bf304838f..33ea1b0c8 100644
--- a/docs/05-database-design.md
+++ b/docs/05-database-design.md
@@ -707,7 +707,7 @@ The data connection also owns:
- **`templates`** — reusable message templates (`src/modules/template/entities/template.entity.ts`), with a unique constraint on `(sessionId, name)` — one template name per session.
- **`status_updates`** — inbound status/story broadcasts with a 24-hour TTL (`src/modules/status-store/entities/status-update.entity.ts`); unique on `(sessionId, waStatusId)`. Attached media is stored via `StorageService`, not in the row.
-- **`webhook_delivery_failures`** — durable record of a webhook delivery that exhausted its retries, or was not sent (attempts 0) (`src/modules/webhook/entities/webhook-delivery-failure.entity.ts`), surfaced via the ADMIN `GET /webhooks/delivery-failures`. A shed or shutdown-refused delivery is replayed by the outbox until its replay budget runs out; an oversize payload or a preflight failure on first dispatch is not replayed. A later successful delivery removes the row.
+- **`webhook_delivery_failures`** — durable record of a webhook delivery that exhausted its retries, or was not sent (attempts 0) (`src/modules/webhook/entities/webhook-delivery-failure.entity.ts`), surfaced via the ADMIN `GET /webhooks/delivery-failures`. A shed or shutdown-refused delivery is replayed by the outbox until its replay budget runs out; an oversize payload or a preflight failure on first dispatch is not replayed. A later successful delivery removes the row. While `WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS` > 0 (default 0), a terminal row (attempts > 0) also keeps the pre-hook event data in `payload` (never selected by the listing, which reports `replayable` instead) so ADMIN `POST /webhooks/delivery-failures/redrive` can replay it with the stored idempotency key; the payload is cleared once that window passes and the row stays.
- **`webhook_outbox_events`**: the outbound webhook delivery record (`src/modules/webhook/entities/webhook-outbox-event.entity.ts`): a row is written `pending` before the delivery attempt, so a delivery lost to a crash is replayed by the reconciler. It settles as `dispatched` once a durable owner (the BullMQ queue, or the inline POST in direct mode) holds it, or as `failed` when the replays run out or the reconciler finds the webhook removed, disabled or unsubscribed from the event (no failure row is written then). Settled rows drop their payload and are pruned on age (§5.7).
- **`plugin_instances`** — one configured instance of an adapter plugin, keyed `${pluginId}:${instanceId}` (`src/modules/integration/entities/plugin-instance.entity.ts`); holds the host-minted ingress HMAC secret, masked on API reads.
- **`ingress_events`** — persist-before-ack durable row and inbound dedup oracle, unique on `(pluginId, instanceId, providerDeliveryId)` (`src/modules/integration/entities/ingress-event.entity.ts`). The full payload is retired to `NULL` once dispatch is settled, leaving a slim dedup marker.
@@ -910,7 +910,8 @@ src/database/migrations/ # data connection (pluggable)
├── 1786700000000-AddMessagesSessionChatCreatedAtIndex.ts # (sessionId, chatId, createdAt): chat thread pages
├── 1786800000000-AddBaileysStoredMessagesSessionCreatedIdIndex.ts # (sessionId, createdAt, id) index for the Baileys store cap trim
├── 1786900000000-ScrubRevokedMessageContent.ts # clears content kept on rows revoked by earlier releases
-└── 1786950000000-ScrubNonPhoneLidMappings.ts # drops lid_mappings rows where earlier releases stored a broadcast id as the phone
+├── 1786950000000-ScrubNonPhoneLidMappings.ts # drops lid_mappings rows where earlier releases stored a broadcast id as the phone
+└── 1787000000000-AddWebhookDeliveryFailurePayload.ts # webhook_delivery_failures.payload (replay copy for the redrive route)
```
> [!NOTE]
@@ -987,7 +988,7 @@ async cleanup(olderThanDays = 30): Promise<number> {
The other interval-based prunes are the same shape: one prune at startup, then a 24-hour `setInterval`, `unref`'d, never a `@Cron`:
-- **`webhook_delivery_failures`** — `WebhookService.onModuleInit()` (`src/modules/webhook/webhook.service.ts`), window `WEBHOOK_FAILURE_RETENTION_DAYS` (default 90; ≤ 0 disables the prune and logs that it is off).
+- **`webhook_delivery_failures`** — `WebhookService.onModuleInit()` (`src/modules/webhook/webhook.service.ts`), window `WEBHOOK_FAILURE_RETENTION_DAYS` (default 90; ≤ 0 disables the prune and logs that it is off). The replay `payload` has its own, shorter window: `WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS` (default 0, store none) clears expired payloads at startup and hourly; at 0 one startup pass clears any payload an earlier setting left.
- **`webhook_outbox_events`**: `WebhookOutboxService.onModuleInit()` (`src/modules/webhook/webhook-outbox.service.ts`), window `WEBHOOK_OUTBOX_RETENTION_DAYS` (default 7). Only settled rows are deleted; a `pending` row is a delivery that can still be replayed and is never pruned on age. A non-positive value does **not** disable the prune: a settled row carries no payload, so the service warns and falls back to 7 days.
- **`ingress_events`** and **`integration_delivery_failures`** — `IntegrationRetentionService` (`src/modules/integration/integration-retention.service.ts`) prunes both in one timer on two independent windows. `INGRESS_DEDUP_RETENTION_DAYS` (default 7) bounds the dedup rows; a non-positive value does **not** disable it — an unpruned dedup table grows without bound for no functional gain, so the service warns and falls back to the 7-day default. `INGRESS_RETENTION_DAYS` (default 90) bounds the DLQ rows, where long retention can be a deliberate operator choice, so ≤ 0 disables that prune (and only that prune).
- **`messages`** and finished **`message_batches`**: `MessageRetentionService` (`src/modules/message/message-retention.service.ts`), window `MESSAGE_RETENTION_DAYS` (default 0, which keeps everything; boot rejects a value above 36500). Messages are deleted by `createdAt` in batches of 500 ids, at most 200 batches per run, so a run deletes at most 100,000 messages; a run that hits that limit logs a warning and is followed by another a minute later until the backlog is cleared. Pending and processing batches are never pruned. Once enabled, send pacing treats a contact silent for longer than the window as a new (cold) contact again, stats totals and search results shrink, archived media files are reclaimed by the chat-media orphan sweep, and a history sync does not write back messages older than the window. Search plugins get no `message:deleted` for pruned rows, so an external index keeps its copies.
diff --git a/docs/06-api-specification.md b/docs/06-api-specification.md
index 4d254e0d1..0188978ba 100644
--- a/docs/06-api-specification.md
+++ b/docs/06-api-specification.md
@@ -4522,7 +4522,7 @@ wire and a client replaying on `503` would publish the status a second time. The
### 6.4.8 Webhooks (management)
-Webhooks are configured per session and managed under `/api/sessions/:sessionId/webhooks` (handled by `WebhookController`). Two cross-session endpoints live on `WebhooksListController`: `GET /api/webhooks` (list, **OPERATOR**) and `GET /api/webhooks/delivery-failures` (dead-letter log, **ADMIN**). Every other route requires an API key with **OPERATOR** role or higher.
+Webhooks are configured per session and managed under `/api/sessions/:sessionId/webhooks` (handled by `WebhookController`). Three cross-session endpoints live on `WebhooksListController`: `GET /api/webhooks` (list, **OPERATOR**), `GET /api/webhooks/delivery-failures` (dead-letter log, **ADMIN**) and `POST /api/webhooks/delivery-failures/redrive` (replay of that log, **ADMIN**). Every other route requires an API key with **OPERATOR** role or higher.
Two fields — `secret` and `headers` — are **write-only**: they are accepted on create/update but are never returned by any webhook route (the response DTO has no `@Expose` for them, so `fromEntity` drops them). `GET /api/infra/export-data` also omits both from its `webhooks` rows, so a backup no longer carries webhook credentials — a restored webhook comes back unsigned (`secret` null, `headers` `{}`) until you set them again. The `secret` is used to compute the `X-OpenWA-Signature: sha256=<hex>` HMAC-SHA256 header on deliveries.
@@ -4660,17 +4660,54 @@ List webhook deliveries that exhausted their retries or were not sent (attempts
"attempts": 4,
"lastStatusCode": 502,
"lastError": "HTTP 502: Bad Gateway",
- "createdAt": "2026-06-25T11:59:00.000Z"
+ "createdAt": "2026-06-25T11:59:00.000Z",
+ "replayable": false
}
]
```
Bare array of `WebhookDeliveryFailure` rows, ordered by `createdAt` descending. `lastStatusCode` is `null` when the failure was a network/timeout/SSRF error rather than a non-2xx response; `idempotencyKey`/`deliveryId` let you correlate the lost event with your own receiver logs.
+`replayable` is `true` when the row still holds the event data and can be replayed with [`POST /api/webhooks/delivery-failures/redrive`](#post-apiwebhooksdelivery-failuresredrive). The event data itself is never returned. Only terminal rows (`attempts` > 0) recorded while `WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS` > 0 keep it, and only until that window passes; with the default `0` every row reads `false`.
+
A delivery shed because the dispatch queue was full, or refused during shutdown, is recorded here with `attempts: 0` while its outbox row stays pending, so the outbox sweep replays it. With the queue disabled, a delivery that shutdown interrupts while it waits out a retry backoff is recorded the same way (`attempts: 0`, `lastError` `ConcurrencyLimiter closed`) if the backoff ends within `WEBHOOK_SHUTDOWN_DRAIN_MS`, although its earlier attempts were sent and their responses are not in the row; one still asleep when the drain ends is only logged (`webhook_delivery_abandoned_shutdown`). Either way its outbox row stays pending, and the replay on the next start replaces or clears the row. Any delivery the receiver accepts with a `2xx`, inline or from a queued job, removes the rows filed under its idempotency key, so a replay that succeeds leaves none. A replay that exhausts its retries files a row carrying the real error and attempt count and only then removes the `attempts: 0` row, so the event stays on record throughout, across a restart or a lost queue job too. A replay that fails before sending (a payload over the size cap after the `webhook:before` hooks, or one that cannot be serialized) gives the `attempts: 0` row that reason instead. A replay that keeps failing files no second row. `openwa_webhook_delivery_failures_total` counts the original shed, and counts once more if a replay also exhausts its retries.
**Errors:** `401` missing/invalid API key · `403` key role below ADMIN
+#### POST /api/webhooks/delivery-failures/redrive
+
+Replay recorded webhook deliveries that still hold their event data, in one bounded batch, oldest first. A row is replayable when it is terminal (`attempts` > 0; an `attempts: 0` row is the outbox's to replay) and was recorded while `WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS` > 0, until that window passes. With the default `0` nothing is kept and this call replays nothing.
+
+Each replay reuses the row's stored `idempotencyKey` (sent as `X-OpenWA-Idempotency-Key`), so a receiver that already handled the event — the POST timed out after it was processed — can dedup the replay instead of acting twice. `webhook:before` hooks run again on the stored pre-hook event data, as on an outbox replay. With the queue disabled each row gets **one** attempt inside the request; with the queue enabled the row is enqueued with the webhook's normal retry policy. A replay that is delivered removes its row; one that fails again keeps the row and raises its `attempts` by one (no second row is filed). Calls on one node run one after another, and every call is written to the audit log with its counts (no payload content).
+
+**Auth:** API key (ADMIN) · **Scope:** rows are confined to the calling key's `allowedSessions`; the body's `sessionId` can only narrow that. A row is replayed only to the webhook it was recorded for, and only while that webhook still belongs to the row's session.
+
+**Request body** (every field optional; an empty body `{}` takes the oldest rows)
+
+| Field | Type | Required | Validation | Description |
+| ----------- | --------------- | -------- | -------------------------------------------- | ------------------------------------------------------------------------- |
+| `sessionId` | string | no | `@IsString`, `@MaxLength(128)` | Only rows of this session. |
+| `webhookId` | string | no | `@IsString`, `@MaxLength(128)` | Only rows of this webhook. |
+| `ids` | string[] | no | `@IsArray`, `@ArrayMaxSize(500)`, each ≤ 128 | Only these failure rows (ids from `GET /api/webhooks/delivery-failures`). |
+| `limit` | integer (1-500) | no | `@IsInt`, `@Min(1)`, `@Max(500)` | Max rows replayed by this call. Defaults to `100`. |
+
+**Response** `200`
+
+```json
+{
+ "redriven": 3,
+ "delivered": 2,
+ "enqueued": 1,
+ "failed": 0,
+ "skipped": 0,
+ "remaining": 0
+}
+```
+
+`redriven` is `delivered` + `enqueued`. `skipped` counts rows not replayed because the webhook was removed, disabled or no longer subscribes to the event, or a `webhook:before` hook cancelled it; those rows stay. `remaining` is the number of replayable rows still in scope after the call — call again while it is above `0`.
+
+**Errors:** `400` body fails validation · `401` missing/invalid API key · `403` key role below ADMIN
+
#### POST /api/sessions/:sessionId/webhooks
Create a webhook for the session.
diff --git a/docs/18-sdk-design.md b/docs/18-sdk-design.md
index 4997ea9b4..9642bb2af 100644
--- a/docs/18-sdk-design.md
+++ b/docs/18-sdk-design.md
@@ -32,7 +32,7 @@ All five SDKs expose the same fluent surface:
| `messages` | list, sendText, sendImage/Video/Audio/Document/Sticker, sendLocation, sendContact, sendTemplate, sendPoll, reply, forward, clickButton, react, delete, editMessage, history, reactions, media, pin, unpin, star, votePoll, sendBulk, batchStatus, cancelBatch |
| `contacts` | list, get, check, profilePicture, profilePictures, phone, upsert, delete, block, unblock, listBlocked |
| `groups` | list, get, create, joinInfo, joinGroup, add/remove/promote/demoteParticipants, setSubject, setDescription, getGroupSettings, updateGroupSettings, leave, getPicture, setPicture, deletePicture, inviteCode, revokeInviteCode, getMembershipRequests, approveMembershipRequests, rejectMembershipRequests |
-| `webhooks` | list, listAll, deliveryFailures, get, create, update, delete, test |
+| `webhooks` | list, listAll, deliveryFailures, redriveDeliveryFailures, get, create, update, delete, test |
| `chats` | list, subscribePresence, getPresence, markRead, markUnread, archive, pin, mute, clearMessages, delete, sendState |
| `labels` | list, get, chats, upsert, delete, forChat, addToChat, removeFromChat _(WhatsApp Business)_ |
| `channels` | list, get, messages, create, delete, mute, subscribe, unsubscribe, demoteAdmin, transferOwnership _(Newsletters)_ |
@@ -243,16 +243,17 @@ Media bodies share the `SendMediaRequest` shape: `{ chatId, url? | base64?, mime
#### `webhooks`
-| Method | Signature | Description |
-| ------------------ | ----------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `listAll` | `listAll(query?)` | List webhooks across EVERY session the key can see, not one session's. **OPERATOR** |
-| `deliveryFailures` | `deliveryFailures(query?)` | Deliveries that exhausted their retries (`attempts > 0`), or were shed, oversize, failed preflight or refused at shutdown, possibly between retries (`attempts: 0`); a later successful delivery removes the row — the diagnostic for a webhook that stopped arriving. A delivery a smart filter suppressed never reaches this log. **ADMIN** |
-| `list` | `list(sessionId)` | List all webhooks for a session. **OPERATOR** |
-| `get` | `get(sessionId, id)` | Get a single webhook by id. **OPERATOR** |
-| `create` | `create(sessionId, body)` | Create a new webhook. **OPERATOR** |
-| `update` | `update(sessionId, id, body)` | Update a webhook. **OPERATOR** |
-| `delete` | `delete(sessionId, id)` | Delete a webhook. **OPERATOR** |
-| `test` | `test(sessionId, id)` | Trigger a test dispatch to the webhook URL and report the result. **OPERATOR** |
+| Method | Signature | Description |
+| ------------------------- | -------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `listAll` | `listAll(query?)` | List webhooks across EVERY session the key can see, not one session's. **OPERATOR** |
+| `deliveryFailures` | `deliveryFailures(query?)` | Deliveries that exhausted their retries (`attempts > 0`), or were shed, oversize, failed preflight or refused at shutdown, possibly between retries (`attempts: 0`); a later successful delivery removes the row — the diagnostic for a webhook that stopped arriving. A delivery a smart filter suppressed never reaches this log. **ADMIN** |
+| `redriveDeliveryFailures` | `redriveDeliveryFailures(body?)` | Replay the rows `deliveryFailures` marks `replayable` (kept while `WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS` > 0), oldest first, one bounded batch per call, reusing each stored idempotency key; returns the delivered/enqueued/failed/skipped/remaining counts. **ADMIN** |
+| `list` | `list(sessionId)` | List all webhooks for a session. **OPERATOR** |
+| `get` | `get(sessionId, id)` | Get a single webhook by id. **OPERATOR** |
+| `create` | `create(sessionId, body)` | Create a new webhook. **OPERATOR** |
+| `update` | `update(sessionId, id, body)` | Update a webhook. **OPERATOR** |
+| `delete` | `delete(sessionId, id)` | Delete a webhook. **OPERATOR** |
+| `test` | `test(sessionId, id)` | Trigger a test dispatch to the webhook URL and report the result. **OPERATOR** |
#### `labels` _(WhatsApp Business)_
@@ -620,16 +621,17 @@ Resources are accessed as properties on the client (e.g. `client.messages`). All
#### `client.webhooks`
-| Method | Signature | Description |
-| ------------------- | --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `list_all` | `list_all(query=None) -> list[WebhookResponse]` | List webhooks across EVERY session the key can see, not one session's. **OPERATOR** |
-| `delivery_failures` | `delivery_failures(query=None) -> list[WebhookDeliveryFailure]` | Deliveries that exhausted their retries (`attempts > 0`), or were shed, oversize, failed preflight or refused at shutdown, possibly between retries (`attempts: 0`); a later successful delivery removes the row — the diagnostic for a webhook that stopped arriving. A delivery a smart filter suppressed never reaches this log. **ADMIN** |
-| `list` | `list(session_id) -> list[WebhookResponse]` | List webhooks. **OPERATOR** |
-| `get` | `get(session_id, webhook_id) -> WebhookResponse` | Get one webhook. **OPERATOR** |
-| `create` | `create(session_id, body) -> WebhookResponse` | Create a webhook. **OPERATOR** |
-| `update` | `update(session_id, webhook_id, body) -> WebhookResponse` | Update a webhook. **OPERATOR** |
-| `delete` | `delete(session_id, webhook_id) -> None` | Delete a webhook. **OPERATOR** |
-| `test` | `test(session_id, webhook_id) -> WebhookTestResult` | Send a test delivery. **OPERATOR** |
+| Method | Signature | Description |
+| --------------------------- | --------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `list_all` | `list_all(query=None) -> list[WebhookResponse]` | List webhooks across EVERY session the key can see, not one session's. **OPERATOR** |
+| `delivery_failures` | `delivery_failures(query=None) -> list[WebhookDeliveryFailure]` | Deliveries that exhausted their retries (`attempts > 0`), or were shed, oversize, failed preflight or refused at shutdown, possibly between retries (`attempts: 0`); a later successful delivery removes the row — the diagnostic for a webhook that stopped arriving. A delivery a smart filter suppressed never reaches this log. **ADMIN** |
+| `redrive_delivery_failures` | `redrive_delivery_failures(body=None) -> WebhookRedriveResult` | Replay the rows `delivery_failures` marks `replayable` (kept while `WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS` > 0), oldest first, one bounded batch per call, reusing each stored idempotency key; returns the delivered/enqueued/failed/skipped/remaining counts. **ADMIN** |
+| `list` | `list(session_id) -> list[WebhookResponse]` | List webhooks. **OPERATOR** |
+| `get` | `get(session_id, webhook_id) -> WebhookResponse` | Get one webhook. **OPERATOR** |
+| `create` | `create(session_id, body) -> WebhookResponse` | Create a webhook. **OPERATOR** |
+| `update` | `update(session_id, webhook_id, body) -> WebhookResponse` | Update a webhook. **OPERATOR** |
+| `delete` | `delete(session_id, webhook_id) -> None` | Delete a webhook. **OPERATOR** |
+| `test` | `test(session_id, webhook_id) -> WebhookTestResult` | Send a test delivery. **OPERATOR** |
#### `client.labels` _(WhatsApp Business)_
@@ -972,16 +974,17 @@ All payloads are associative arrays; all listed methods are synchronous and retu
#### `webhooks`
-| Method | Signature | Description |
-| ------------------ | ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `listAll` | `listAll(array $query = []): array` | List webhooks across EVERY session the key can see, not one session's. **OPERATOR** |
-| `deliveryFailures` | `deliveryFailures(array $query = [])` | Deliveries that exhausted their retries (`attempts > 0`), or were shed, oversize, failed preflight or refused at shutdown, possibly between retries (`attempts: 0`); a later successful delivery removes the row — the diagnostic for a webhook that stopped arriving. A delivery a smart filter suppressed never reaches this log. **ADMIN** |
-| `list` | `list(string $sessionId): array` | List webhooks. **OPERATOR** |
-| `get` | `get(string $sessionId, string $id): array` | Get one webhook. **OPERATOR** |
-| `create` | `create(string $sessionId, array $body): array` | Create a webhook. **OPERATOR** |
-| `update` | `update(string $sessionId, string $id, array $body): array` | Update a webhook. **OPERATOR** |
-| `delete` | `delete(string $sessionId, string $id): void` | Delete a webhook. **OPERATOR** |
-| `test` | `test(string $sessionId, string $id): array` | Send a test event. **OPERATOR** |
+| Method | Signature | Description |
+| ------------------------- | ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `listAll` | `listAll(array $query = []): array` | List webhooks across EVERY session the key can see, not one session's. **OPERATOR** |
+| `deliveryFailures` | `deliveryFailures(array $query = [])` | Deliveries that exhausted their retries (`attempts > 0`), or were shed, oversize, failed preflight or refused at shutdown, possibly between retries (`attempts: 0`); a later successful delivery removes the row — the diagnostic for a webhook that stopped arriving. A delivery a smart filter suppressed never reaches this log. **ADMIN** |
+| `redriveDeliveryFailures` | `redriveDeliveryFailures(array $body = []): array` | Replay the rows `deliveryFailures` marks `replayable` (kept while `WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS` > 0), oldest first, one bounded batch per call, reusing each stored idempotency key; returns the delivered/enqueued/failed/skipped/remaining counts. **ADMIN** |
+| `list` | `list(string $sessionId): array` | List webhooks. **OPERATOR** |
+| `get` | `get(string $sessionId, string $id): array` | Get one webhook. **OPERATOR** |
+| `create` | `create(string $sessionId, array $body): array` | Create a webhook. **OPERATOR** |
+| `update` | `update(string $sessionId, string $id, array $body): array` | Update a webhook. **OPERATOR** |
+| `delete` | `delete(string $sessionId, string $id): void` | Delete a webhook. **OPERATOR** |
+| `test` | `test(string $sessionId, string $id): array` | Send a test event. **OPERATOR** |
#### `labels` _(WhatsApp Business)_
diff --git a/openapi.json b/openapi.json
index 672da60b2..9ed90cb29 100644
--- a/openapi.json
+++ b/openapi.json
@@ -85,6 +85,7 @@
"webhook_deleted",
"webhook_triggered",
"webhook_failed",
+ "webhook_deliveries_redriven",
"integration_instance_created",
"integration_instance_updated",
"integration_instance_secret_regenerated",
@@ -3708,6 +3709,68 @@
]
}
},
+ "/api/webhooks/delivery-failures/redrive": {
+ "post": {
+ "operationId": "WebhooksListController_redriveDeliveryFailures",
+ "parameters": [],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/RedriveWebhookDeliveriesDto"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "One bounded batch of replayable failure rows (terminal rows recorded while WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS > 0), oldest first. Each replay reuses the stored idempotency key, so a receiver that already handled the event can dedup it, and runs the webhook:before hooks again. With the queue disabled each row gets one attempt; with it enabled the row is enqueued with the webhook retry policy. Rows outside the key allowedSessions are never touched.",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/WebhookRedriveResultDto"
+ }
+ }
+ }
+ },
+ "400": {
+ "description": "The body is malformed, fails validation, or carries a field the DTO does not declare",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "401": {
+ "description": "The API key is missing, unknown, revoked, or expired",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "403": {
+ "description": "The API key's role, session or chat scope, or IP allow-list does not allow this operation",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ }
+ },
+ "summary": "Replay recorded webhook deliveries that still hold their event data",
+ "tags": [
+ "webhooks"
+ ]
+ }
+ },
"/api/webhooks": {
"get": {
"operationId": "WebhooksListController_findAll",
@@ -19120,6 +19183,11 @@
"type": "string",
"format": "date-time",
"description": "When the failure was first recorded."
+ },
+ "replayable": {
+ "type": "boolean",
+ "description": "True when the row still holds the event data and can be replayed with `POST /webhooks/delivery-failures/redrive`. Only terminal rows (attempts > 0) recorded while `WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS` > 0 are, until that window passes.",
+ "example": false
}
},
"required": [
@@ -19130,7 +19198,81 @@
"url",
"attempts",
"lastError",
- "createdAt"
+ "createdAt",
+ "replayable"
+ ]
+ },
+ "RedriveWebhookDeliveriesDto": {
+ "type": "object",
+ "properties": {
+ "sessionId": {
+ "type": "string",
+ "description": "Only rows of this session. Narrows within the calling key's allowedSessions, never past it.",
+ "example": "my-session"
+ },
+ "webhookId": {
+ "type": "string",
+ "description": "Only rows of this webhook.",
+ "example": "0a941dac-a965-45e7-b318-74ae8be134f0"
+ },
+ "ids": {
+ "description": "Only these failure rows (ids from `GET /webhooks/delivery-failures`).",
+ "example": [
+ "0a941dac-a965-45e7-b318-74ae8be134f0"
+ ],
+ "type": "array",
+ "items": {
+ "type": "string"
+ }
+ },
+ "limit": {
+ "type": "number",
+ "description": "Max rows replayed by this call (1-500, default 100).",
+ "example": 100
+ }
+ }
+ },
+ "WebhookRedriveResultDto": {
+ "type": "object",
+ "properties": {
+ "redriven": {
+ "type": "number",
+ "description": "Rows replayed by this call: delivered plus enqueued.",
+ "example": 3
+ },
+ "delivered": {
+ "type": "number",
+ "description": "Delivered by a direct POST; their failure rows were removed.",
+ "example": 2
+ },
+ "enqueued": {
+ "type": "number",
+ "description": "Handed to the queue; each row is removed when its job delivers and kept if it fails again.",
+ "example": 1
+ },
+ "failed": {
+ "type": "number",
+ "description": "Replays that failed again; their rows stay, with attempts raised by one.",
+ "example": 0
+ },
+ "skipped": {
+ "type": "number",
+ "description": "Rows not replayed: the webhook was removed, disabled or unsubscribed, or a plugin cancelled it.",
+ "example": 0
+ },
+ "remaining": {
+ "type": "number",
+ "description": "Replayable rows still in scope after this call.",
+ "example": 0
+ }
+ },
+ "required": [
+ "redriven",
+ "delivered",
+ "enqueued",
+ "failed",
+ "skipped",
+ "remaining"
]
},
"CreateAutomationRuleDto": {
diff --git a/scripts/check-contract-shapes.mjs b/scripts/check-contract-shapes.mjs
index dec74a398..fc14f36cc 100644
--- a/scripts/check-contract-shapes.mjs
+++ b/scripts/check-contract-shapes.mjs
@@ -137,9 +137,12 @@ const MAPPINGS = {
UpdateSessionProxyRequest: 'UpdateSessionProxyDto',
UpsertContactRequest: 'UpsertContactDto',
UpsertLabelRequest: 'UpsertLabelDto',
+ RedriveWebhookDeliveriesRequest: 'RedriveWebhookDeliveriesDto',
+ RedriveWebhookDeliveriesRequest: 'RedriveWebhookDeliveriesDto',
VotePollRequest: 'VotePollDto',
WebhookDeliveryFailure: 'WebhookDeliveryFailureDto',
WebhookFilterCondition: 'WebhookFilterConditionDto',
+ WebhookRedriveResult: 'WebhookRedriveResultDto',
WebhookResponse: 'WebhookResponseDto',
},
'dashboard/src/services/api.ts': {
@@ -293,8 +296,10 @@ const PYTHON_MAPPING = {
UpdateWebhookRequest: 'UpdateWebhookDto',
UpsertContactRequest: 'UpsertContactDto',
UpsertLabelRequest: 'UpsertLabelDto',
+ RedriveWebhookDeliveriesRequest: 'RedriveWebhookDeliveriesDto',
VotePollRequest: 'VotePollDto',
WebhookDeliveryFailure: 'WebhookDeliveryFailureDto',
+ WebhookRedriveResult: 'WebhookRedriveResultDto',
WebhookResponse: 'WebhookResponseDto',
};
@@ -378,8 +383,10 @@ const GO_MAPPING = {
UpdateWebhookRequest: 'UpdateWebhookDto',
UpsertContactRequest: 'UpsertContactDto',
UpsertLabelRequest: 'UpsertLabelDto',
+ RedriveWebhookDeliveriesRequest: 'RedriveWebhookDeliveriesDto',
VotePollRequest: 'VotePollDto',
WebhookDeliveryFailure: 'WebhookDeliveryFailureDto',
+ WebhookRedriveResult: 'WebhookRedriveResultDto',
WebhookResponse: 'WebhookResponseDto',
};
@@ -467,8 +474,10 @@ const JAVA_MAPPING = {
UpdateWebhookRequest: 'UpdateWebhookDto',
UpsertContactRequest: 'UpsertContactDto',
UpsertLabelRequest: 'UpsertLabelDto',
+ RedriveWebhookDeliveriesRequest: 'RedriveWebhookDeliveriesDto',
VotePollRequest: 'VotePollDto',
WebhookDeliveryFailure: 'WebhookDeliveryFailureDto',
+ WebhookRedriveResult: 'WebhookRedriveResultDto',
WebhookResponse: 'WebhookResponseDto',
};
diff --git a/sdk/README.md b/sdk/README.md
index ed0f8d421..d3a46dbb3 100644
--- a/sdk/README.md
+++ b/sdk/README.md
@@ -30,7 +30,7 @@ All five SDKs expose the same fluent resource surface:
| `messages` | list, sendText, sendImage/Video/Audio/Document/Sticker, sendLocation, sendContact, sendTemplate, sendPoll, reply, clickButton, forward, react, delete, editMessage, history, reactions, media, pin, unpin, star, votePoll, sendBulk, batchStatus, cancelBatch |
| `contacts` | list, get, check, profilePicture, profilePictures, phone, upsert, delete, block, unblock, listBlocked |
| `groups` | list, get, create, joinGroup, joinInfo, add/remove/promote/demoteParticipants, setSubject, setDescription, get/updateGroupSettings, leave, getPicture, setPicture, deletePicture, inviteCode, revokeInviteCode, getMembershipRequests, approveMembershipRequests, rejectMembershipRequests |
-| `webhooks` | list, listAll, deliveryFailures, get, create, update, delete, test |
+| `webhooks` | list, listAll, deliveryFailures, redriveDeliveryFailures, get, create, update, delete, test |
| `chats` | list, markRead, markUnread, archive, pin, mute, clearMessages, delete, sendState, subscribePresence, getPresence |
| `labels` | list, get, chats, forChat, upsert, delete, addToChat, removeFromChat _(WhatsApp Business)_ |
| `channels` | list, get, messages, create, delete, mute, subscribe, unsubscribe, demoteAdmin, transferOwnership _(Newsletters)_ |
diff --git a/sdk/go/routing_test.go b/sdk/go/routing_test.go
index 245c364cf..adbe37923 100644
--- a/sdk/go/routing_test.go
+++ b/sdk/go/routing_test.go
@@ -101,6 +101,7 @@ func TestRouting(t *testing.T) {
{"Webhooks.Test", func(c *Client) { c.Webhooks.Test(ctx, "s1", "w1") }, "POST", "/api/sessions/s1/webhooks/w1/test"},
{"Webhooks.ListAll", func(c *Client) { c.Webhooks.ListAll(ctx, nil) }, "GET", "/api/webhooks"},
{"Webhooks.DeliveryFailures", func(c *Client) { c.Webhooks.DeliveryFailures(ctx, nil) }, "GET", "/api/webhooks/delivery-failures"},
+ {"Webhooks.RedriveDeliveryFailures", func(c *Client) { c.Webhooks.RedriveDeliveryFailures(ctx, nil) }, "POST", "/api/webhooks/delivery-failures/redrive"},
{"Media.ConversionStatus", func(c *Client) { c.Media.ConversionStatus(ctx, "s1") }, "GET", "/api/sessions/s1/media/convert"},
{"Media.ConvertVoice", func(c *Client) { c.Media.ConvertVoice(ctx, "s1", ConvertMediaInput{}) }, "POST", "/api/sessions/s1/media/convert/voice"},
diff --git a/sdk/go/types_webhook.go b/sdk/go/types_webhook.go
index 919356e4b..104843180 100644
--- a/sdk/go/types_webhook.go
+++ b/sdk/go/types_webhook.go
@@ -203,6 +203,42 @@ type WebhookDeliveryFailure struct {
LastError string `json:"lastError"`
// CreatedAt is the ISO timestamp of when the failure was first recorded.
CreatedAt string `json:"createdAt"`
+ // Replayable is true when the row still holds the event data and can be
+ // replayed with RedriveDeliveryFailures: a terminal row (Attempts > 0)
+ // recorded while the gateway's WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS is
+ // above 0, until that window passes.
+ Replayable bool `json:"replayable"`
+}
+
+// RedriveWebhookDeliveriesRequest is the body of RedriveDeliveryFailures.
+// Every field narrows; an empty body takes the oldest rows.
+type RedriveWebhookDeliveriesRequest struct {
+ // SessionID limits the batch to one session (within the key's allowedSessions).
+ SessionID string `json:"sessionId,omitempty"`
+ // WebhookID limits the batch to one webhook.
+ WebhookID string `json:"webhookId,omitempty"`
+ // IDs limits the batch to these failure rows (at most 500).
+ IDs []string `json:"ids,omitempty"`
+ // Limit caps the rows replayed by this call (1-500, default 100).
+ Limit int `json:"limit,omitempty"`
+}
+
+// WebhookRedriveResult is the outcome of RedriveDeliveryFailures.
+type WebhookRedriveResult struct {
+ // Redriven is the rows replayed by this call: Delivered plus Enqueued.
+ Redriven int `json:"redriven"`
+ // Delivered rows went out by a direct POST; their failure rows were removed.
+ Delivered int `json:"delivered"`
+ // Enqueued rows were handed to the queue; each is removed when its job
+ // delivers and kept if it fails again.
+ Enqueued int `json:"enqueued"`
+ // Failed replays failed again; their rows stay, with attempts raised by one.
+ Failed int `json:"failed"`
+ // Skipped rows were not replayed: the webhook was removed, disabled or
+ // unsubscribed, or a plugin cancelled it.
+ Skipped int `json:"skipped"`
+ // Remaining is the replayable rows still in scope after this call.
+ Remaining int `json:"remaining"`
}
// WebhookDelivery is the JSON body of a webhook delivery (docs/06 section
diff --git a/sdk/go/webhooks.go b/sdk/go/webhooks.go
index 3b923d83b..1123711ad 100644
--- a/sdk/go/webhooks.go
+++ b/sdk/go/webhooks.go
@@ -32,6 +32,23 @@ func (s *WebhooksService) DeliveryFailures(ctx context.Context, query *DeliveryF
return out, err
}
+// RedriveDeliveryFailures replays recorded deliveries that still hold their event data
+// (Replayable in DeliveryFailures), oldest first, one bounded batch per call. Each replay reuses the
+// stored idempotency key, so a receiver that already handled the event can dedup it. Only rows
+// recorded while the gateway's WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS is above 0 are replayable.
+// Requires an ADMIN-level key; rows outside the key's allowedSessions are never touched. A nil body
+// takes the oldest rows.
+func (s *WebhooksService) RedriveDeliveryFailures(ctx context.Context, body *RedriveWebhookDeliveriesRequest) (*WebhookRedriveResult, error) {
+ if body == nil {
+ body = &RedriveWebhookDeliveriesRequest{}
+ }
+ var out WebhookRedriveResult
+ if err := s.client.do(ctx, "POST", "/api/webhooks/delivery-failures/redrive", nil, body, &out); err != nil {
+ return nil, err
+ }
+ return &out, nil
+}
+
// List returns webhooks for a session.
func (s *WebhooksService) List(ctx context.Context, sessionID string) ([]WebhookResponse, error) {
var out []WebhookResponse
diff --git a/sdk/java/src/main/java/com/rmyndharis/openwa/model/RedriveWebhookDeliveriesRequest.java b/sdk/java/src/main/java/com/rmyndharis/openwa/model/RedriveWebhookDeliveriesRequest.java
new file mode 100644
index 000000000..23f206a1f
--- /dev/null
+++ b/sdk/java/src/main/java/com/rmyndharis/openwa/model/RedriveWebhookDeliveriesRequest.java
@@ -0,0 +1,48 @@
+package com.rmyndharis.openwa.model;
+
+import java.util.List;
+
+/**
+ * Request body for replaying recorded webhook deliveries. Every field narrows; {@code null} fields
+ * are omitted, and an empty body takes the oldest rows.
+ */
+public record RedriveWebhookDeliveriesRequest(String sessionId, String webhookId, List<String> ids, Integer limit) {
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static final class Builder {
+ private String sessionId;
+ private String webhookId;
+ private List<String> ids;
+ private Integer limit;
+
+ /** Only rows of this session (within the key's allowedSessions). */
+ public Builder sessionId(String v) {
+ this.sessionId = v;
+ return this;
+ }
+
+ /** Only rows of this webhook. */
+ public Builder webhookId(String v) {
+ this.webhookId = v;
+ return this;
+ }
+
+ /** Only these failure rows (ids from {@code deliveryFailures}), at most 500. */
+ public Builder ids(List<String> v) {
+ this.ids = v;
+ return this;
+ }
+
+ /** Max rows replayed by this call (1-500, default 100). */
+ public Builder limit(Integer v) {
+ this.limit = v;
+ return this;
+ }
+
+ public RedriveWebhookDeliveriesRequest build() {
+ return new RedriveWebhookDeliveriesRequest(sessionId, webhookId, ids, limit);
+ }
+ }
+}
diff --git a/sdk/java/src/main/java/com/rmyndharis/openwa/model/WebhookDeliveryFailure.java b/sdk/java/src/main/java/com/rmyndharis/openwa/model/WebhookDeliveryFailure.java
index 27ffc505a..a4f44d1c8 100644
--- a/sdk/java/src/main/java/com/rmyndharis/openwa/model/WebhookDeliveryFailure.java
+++ b/sdk/java/src/main/java/com/rmyndharis/openwa/model/WebhookDeliveryFailure.java
@@ -27,4 +27,10 @@ public record WebhookDeliveryFailure(
Integer lastStatusCode,
String lastError,
/** ISO timestamp of when the failure was recorded. */
- String createdAt) {}
+ String createdAt,
+ /**
+ * True when the row still holds the event data and can be replayed with {@code
+ * redriveDeliveryFailures}: a terminal row (attempts &gt; 0) recorded while the gateway's
+ * WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS is above 0, until that window passes.
+ */
+ boolean replayable) {}
diff --git a/sdk/java/src/main/java/com/rmyndharis/openwa/model/WebhookRedriveResult.java b/sdk/java/src/main/java/com/rmyndharis/openwa/model/WebhookRedriveResult.java
new file mode 100644
index 000000000..aa5680ade
--- /dev/null
+++ b/sdk/java/src/main/java/com/rmyndharis/openwa/model/WebhookRedriveResult.java
@@ -0,0 +1,16 @@
+package com.rmyndharis.openwa.model;
+
+/**
+ * Outcome of replaying recorded webhook deliveries.
+ *
+ * @param redriven rows replayed by this call: delivered plus enqueued
+ * @param delivered delivered by a direct POST; their failure rows were removed
+ * @param enqueued handed to the queue; each row is removed when its job delivers and kept if it
+ * fails again
+ * @param failed replays that failed again; their rows stay, with attempts raised by one
+ * @param skipped rows not replayed: the webhook was removed, disabled or unsubscribed, or a plugin
+ * cancelled it
+ * @param remaining replayable rows still in scope after this call
+ */
+public record WebhookRedriveResult(
+ int redriven, int delivered, int enqueued, int failed, int skipped, int remaining) {}
diff --git a/sdk/java/src/main/java/com/rmyndharis/openwa/resources/WebhooksResource.java b/sdk/java/src/main/java/com/rmyndharis/openwa/resources/WebhooksResource.java
index 295d40d93..ab5456fbf 100644
--- a/sdk/java/src/main/java/com/rmyndharis/openwa/resources/WebhooksResource.java
+++ b/sdk/java/src/main/java/com/rmyndharis/openwa/resources/WebhooksResource.java
@@ -6,8 +6,10 @@ import com.rmyndharis.openwa.OpenWAClient;
import com.rmyndharis.openwa.http.HttpMethod;
import com.rmyndharis.openwa.model.DeliveryFailureQuery;
import com.rmyndharis.openwa.model.CreateWebhookRequest;
+import com.rmyndharis.openwa.model.RedriveWebhookDeliveriesRequest;
import com.rmyndharis.openwa.model.UpdateWebhookRequest;
import com.rmyndharis.openwa.model.WebhookDeliveryFailure;
+import com.rmyndharis.openwa.model.WebhookRedriveResult;
import com.rmyndharis.openwa.model.WebhookResponse;
import com.rmyndharis.openwa.model.WebhookTestResult;
import java.util.List;
@@ -43,6 +45,24 @@ public final class WebhooksResource {
HttpMethod.GET, "/api/webhooks/delivery-failures", query, null, WebhookDeliveryFailure.class);
}
+ /**
+ * Replay recorded deliveries that still hold their event data ({@code replayable} in {@link
+ * #deliveryFailures}), oldest first, one bounded batch per call. Each replay reuses the stored
+ * idempotency key, so a receiver that already handled the event can dedup it. Only rows recorded
+ * while the gateway's WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS is above 0 are replayable. Requires
+ * an ADMIN-level key; rows outside the key's allowedSessions are never touched.
+ *
+ * @param body the batch filter; {@code null} takes the oldest rows
+ */
+ public WebhookRedriveResult redriveDeliveryFailures(RedriveWebhookDeliveriesRequest body) {
+ return client.request(
+ HttpMethod.POST,
+ "/api/webhooks/delivery-failures/redrive",
+ null,
+ body != null ? body : RedriveWebhookDeliveriesRequest.builder().build(),
+ WebhookRedriveResult.class);
+ }
+
/** List all webhooks for a session. */
public List<WebhookResponse> list(String sessionId) {
return client.requestList(
diff --git a/sdk/java/src/test/java/com/rmyndharis/openwa/resources/WebhooksResourceTest.java b/sdk/java/src/test/java/com/rmyndharis/openwa/resources/WebhooksResourceTest.java
index 1470a53f3..3e8509662 100644
--- a/sdk/java/src/test/java/com/rmyndharis/openwa/resources/WebhooksResourceTest.java
+++ b/sdk/java/src/test/java/com/rmyndharis/openwa/resources/WebhooksResourceTest.java
@@ -7,10 +7,12 @@ import com.rmyndharis.openwa.ClientConfig;
import com.rmyndharis.openwa.OpenWAClient;
import com.rmyndharis.openwa.http.HttpMethod;
import com.rmyndharis.openwa.model.CreateWebhookRequest;
+import com.rmyndharis.openwa.model.RedriveWebhookDeliveriesRequest;
import com.rmyndharis.openwa.model.UpdateWebhookRequest;
import com.rmyndharis.openwa.model.WebhookEvent;
import com.rmyndharis.openwa.model.WebhookFilterCondition;
import com.rmyndharis.openwa.model.WebhookFilters;
+import com.rmyndharis.openwa.model.WebhookRedriveResult;
import com.rmyndharis.openwa.support.MockTransport;
import java.util.List;
import org.junit.jupiter.api.Test;
@@ -24,6 +26,23 @@ class WebhooksResourceTest {
final OpenWAClient client = new OpenWAClient(
ClientConfig.builder().baseUrl("http://h").apiKey("k").transport(tx).build());
+ @Test
+ void redriveDeliveryFailuresPostsTheFilterOrAnEmptyBody() {
+ String result =
+ "{\"redriven\":1,\"delivered\":1,\"enqueued\":0,\"failed\":0,\"skipped\":0,\"remaining\":3}";
+ tx.respond(200, result);
+ WebhookRedriveResult out =
+ client.webhooks.redriveDeliveryFailures(
+ RedriveWebhookDeliveriesRequest.builder().sessionId("s").limit(10).build());
+ assertEquals("http://h/api/webhooks/delivery-failures/redrive", tx.lastRequest().url());
+ assertEquals(HttpMethod.POST, tx.lastRequest().method());
+ assertEquals("{\"sessionId\":\"s\",\"limit\":10}", tx.lastRequest().body());
+ assertEquals(3, out.remaining());
+ tx.respond(200, result);
+ client.webhooks.redriveDeliveryFailures(null);
+ assertEquals("{}", tx.lastRequest().body());
+ }
+
@Test
void listHitsWebhooksRoot() {
tx.respond(200, "[]");
diff --git a/sdk/javascript/src/resources/webhooks.ts b/sdk/javascript/src/resources/webhooks.ts
index b3b217b33..df1d22920 100644
--- a/sdk/javascript/src/resources/webhooks.ts
+++ b/sdk/javascript/src/resources/webhooks.ts
@@ -9,8 +9,10 @@ import { encodeSegment } from '../http.js';
import type { OpenWAClient } from '../client.js';
import type {
CreateWebhookRequest,
+ RedriveWebhookDeliveriesRequest,
UpdateWebhookRequest,
WebhookDeliveryFailure,
+ WebhookRedriveResult,
WebhookResponse,
WebhookTestResult,
} from '../types.js';
@@ -55,6 +57,21 @@ export class WebhooksResource {
});
}
+ /**
+ * Replay recorded deliveries that still hold their event data (`replayable: true` in
+ * {@link deliveryFailures}), oldest first, one bounded batch per call. Each replay reuses the stored
+ * idempotency key, so a receiver that already handled the event can dedup it. Only rows recorded
+ * while the gateway's `WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS` is above 0 are replayable. Requires
+ * an ADMIN-level key; rows outside the key's allowedSessions are never touched.
+ */
+ redriveDeliveryFailures(body: RedriveWebhookDeliveriesRequest = {}): Promise<WebhookRedriveResult> {
+ return this.client.request<WebhookRedriveResult>({
+ method: 'POST',
+ path: '/api/webhooks/delivery-failures/redrive',
+ body,
+ });
+ }
+
/** List all webhooks for a session. */
list(sessionId: string): Promise<WebhookResponse[]> {
return this.client.request<WebhookResponse[]>({
diff --git a/sdk/javascript/src/types.ts b/sdk/javascript/src/types.ts
index c1a74fd7c..e7969732b 100644
--- a/sdk/javascript/src/types.ts
+++ b/sdk/javascript/src/types.ts
@@ -1030,6 +1030,40 @@ export interface WebhookDeliveryFailure {
lastError: string;
/** ISO timestamp of when the row was recorded. */
createdAt: string;
+ /**
+ * True when the row still holds the event data and can be replayed with
+ * {@link WebhooksResource.redriveDeliveryFailures}: a terminal row (`attempts > 0`) recorded while the
+ * gateway's `WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS` is above 0, until that window passes.
+ */
+ replayable: boolean;
+}
+
+/** Body of {@link WebhooksResource.redriveDeliveryFailures}. Every field narrows; an empty body takes the oldest rows. */
+export interface RedriveWebhookDeliveriesRequest {
+ /** Only rows of this session (within the key's allowedSessions). */
+ sessionId?: string;
+ /** Only rows of this webhook. */
+ webhookId?: string;
+ /** Only these failure rows (ids from `deliveryFailures`), at most 500. */
+ ids?: string[];
+ /** Max rows replayed by this call (1-500, default 100). */
+ limit?: number;
+}
+
+/** Outcome of {@link WebhooksResource.redriveDeliveryFailures}. */
+export interface WebhookRedriveResult {
+ /** Rows replayed by this call: delivered plus enqueued. */
+ redriven: number;
+ /** Delivered by a direct POST; their failure rows were removed. */
+ delivered: number;
+ /** Handed to the queue; each row is removed when its job delivers and kept if it fails again. */
+ enqueued: number;
+ /** Replays that failed again; their rows stay, with attempts raised by one. */
+ failed: number;
+ /** Rows not replayed: the webhook was removed, disabled or unsubscribed, or a plugin cancelled it. */
+ skipped: number;
+ /** Replayable rows still in scope after this call. */
+ remaining: number;
}
// ── Chat (session-scoped chat operations) ─────────────────────────
diff --git a/sdk/javascript/test/resources.test.ts b/sdk/javascript/test/resources.test.ts
index 51f23299c..9b0e8f146 100644
--- a/sdk/javascript/test/resources.test.ts
+++ b/sdk/javascript/test/resources.test.ts
@@ -281,6 +281,16 @@ describe('WebhooksResource — exact paths', () => {
await client(t).webhooks.create('s', { url: 'u', events: ['message.received'], filters });
expect(t.lastCall!.body).toEqual({ url: 'u', events: ['message.received'], filters });
});
+
+ it('redriveDeliveryFailures POSTs the filter (or an empty body) to the redrive route', async () => {
+ const result = { redriven: 1, delivered: 1, enqueued: 0, failed: 0, skipped: 0, remaining: 0 };
+ const t = new MockTransport().on('POST', /\/api\/webhooks\/delivery-failures\/redrive$/, { body: result });
+ const c = client(t);
+ await expect(c.webhooks.redriveDeliveryFailures({ sessionId: 's', limit: 10 })).resolves.toEqual(result);
+ expect(t.lastCall!.body).toEqual({ sessionId: 's', limit: 10 });
+ await c.webhooks.redriveDeliveryFailures();
+ expect(t.lastCall!.body).toEqual({});
+ });
});
describe('StatusResource — nested media bodies', () => {
diff --git a/sdk/php/src/Resources/WebhooksResource.php b/sdk/php/src/Resources/WebhooksResource.php
index 7dfef2322..6c134843c 100644
--- a/sdk/php/src/Resources/WebhooksResource.php
+++ b/sdk/php/src/Resources/WebhooksResource.php
@@ -47,13 +47,31 @@ class WebhooksResource
* @return array<int,array<string,mixed>> most recent first; each entry carries `id`, `webhookId`,
* `sessionId`, `event`, `url`, `attempts`, `lastError` and
* `createdAt`, plus nullable `idempotencyKey`, `deliveryId`
- * and `lastStatusCode`
+ * and `lastStatusCode`, and `replayable` (true when the row
+ * can be replayed with redriveDeliveryFailures)
*/
public function deliveryFailures(array $query = [])
{
return $this->http->request('GET', '/api/webhooks/delivery-failures', $query);
}
+ /**
+ * Replay recorded deliveries that still hold their event data (`replayable` in deliveryFailures),
+ * oldest first, one bounded batch per call. Each replay reuses the stored idempotency key, so a
+ * receiver that already handled the event can dedup it. Only rows recorded while the gateway's
+ * WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS is above 0 are replayable. Requires an ADMIN-level key;
+ * rows outside the key's allowedSessions are never touched.
+ *
+ * @param array<string,mixed> $body Optional filter: `sessionId`, `webhookId`, `ids` (at most 500),
+ * `limit` (1-500, default 100). Empty takes the oldest rows.
+ *
+ * @return array<string,mixed> `redriven`, `delivered`, `enqueued`, `failed`, `skipped`, `remaining`
+ */
+ public function redriveDeliveryFailures(array $body = []): array
+ {
+ return $this->http->request('POST', '/api/webhooks/delivery-failures/redrive', [], (object) $body);
+ }
+
/** @return array<int,array<string,mixed>> */
public function list(string $sessionId): array
{
diff --git a/sdk/php/tests/ResourcesTest.php b/sdk/php/tests/ResourcesTest.php
index 381c88573..7c4c7944f 100644
--- a/sdk/php/tests/ResourcesTest.php
+++ b/sdk/php/tests/ResourcesTest.php
@@ -562,6 +562,23 @@ class ResourcesTest extends TestCase
$this->assertStringContainsString('call.whatsapp.com', $res['link']);
}
+ public function testRedriveDeliveryFailuresPostsTheFilterOrAnEmptyObject(): void
+ {
+ $result = ['redriven' => 1, 'delivered' => 1, 'enqueued' => 0, 'failed' => 0, 'skipped' => 0, 'remaining' => 0];
+ $backend = new MockBackend();
+ $backend->on(200, $result);
+ $backend->on(200, $result);
+ $client = $backend->makeClient();
+
+ $this->assertSame($result, $client->webhooks->redriveDeliveryFailures(['sessionId' => 's', 'limit' => 10]));
+ $this->assertRequest($backend, 'POST', '/api/webhooks/delivery-failures/redrive');
+ $this->assertSame('{"sessionId":"s","limit":10}', $backend->rawBody(0));
+
+ // An empty filter must go out as a JSON object: `[]` is not a valid body for the DTO.
+ $client->webhooks->redriveDeliveryFailures();
+ $this->assertSame('{}', $backend->rawBody(1));
+ }
+
private function assertRequest(MockBackend $backend, string $method, string $path): void
{
$this->assertSame($method, $backend->lastCall()['method']);
diff --git a/sdk/python/openwa/resources/webhooks.py b/sdk/python/openwa/resources/webhooks.py
index 7a0accf67..c195f2ef0 100644
--- a/sdk/python/openwa/resources/webhooks.py
+++ b/sdk/python/openwa/resources/webhooks.py
@@ -10,8 +10,10 @@ from typing import TYPE_CHECKING, TypedDict
from .._http import quote_segment
from ..types import (
CreateWebhookRequest,
+ RedriveWebhookDeliveriesRequest,
UpdateWebhookRequest,
WebhookDeliveryFailure,
+ WebhookRedriveResult,
WebhookResponse,
WebhookTestResult,
)
@@ -57,6 +59,16 @@ class WebhooksResource:
"""
return self._http.request("GET", "/api/webhooks/delivery-failures", query=query)
+ def redrive_delivery_failures(self, body: RedriveWebhookDeliveriesRequest | None = None) -> WebhookRedriveResult:
+ """Replay recorded deliveries that still hold their event data, oldest first, one bounded batch per call.
+
+ Only rows with ``replayable: True`` in :meth:`delivery_failures` qualify: terminal rows recorded while
+ the gateway's ``WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS`` is above 0. Each replay reuses the stored
+ idempotency key, so a receiver that already handled the event can dedup it. Requires an ADMIN-level
+ key; rows outside the key's allowedSessions are never touched.
+ """
+ return self._http.request("POST", "/api/webhooks/delivery-failures/redrive", body=body or {})
+
def list(self, session_id: str) -> list[WebhookResponse]:
return self._http.request("GET", f"/api/sessions/{quote_segment(session_id)}/webhooks")
diff --git a/sdk/python/openwa/types.py b/sdk/python/openwa/types.py
index 3473f33dd..1e82ac2b0 100644
--- a/sdk/python/openwa/types.py
+++ b/sdk/python/openwa/types.py
@@ -993,6 +993,40 @@ class WebhookDeliveryFailure(TypedDict):
lastError: str
# ISO timestamp of when the failure was first recorded.
createdAt: str
+ # True when the row still holds the event data and can be replayed with
+ # WebhooksResource.redrive_delivery_failures: a terminal row (attempts > 0) recorded while the
+ # gateway's WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS is above 0, until that window passes.
+ replayable: bool
+
+
+class RedriveWebhookDeliveriesRequest(TypedDict, total=False):
+ """Body of ``WebhooksResource.redrive_delivery_failures``. Every field narrows; an empty body takes the oldest rows."""
+
+ # Only rows of this session (within the key's allowedSessions).
+ sessionId: str
+ # Only rows of this webhook.
+ webhookId: str
+ # Only these failure rows (ids from delivery_failures), at most 500.
+ ids: list[str]
+ # Max rows replayed by this call (1-500, default 100).
+ limit: int
+
+
+class WebhookRedriveResult(TypedDict):
+ """Outcome of ``WebhooksResource.redrive_delivery_failures``."""
+
+ # Rows replayed by this call: delivered plus enqueued.
+ redriven: int
+ # Delivered by a direct POST; their failure rows were removed.
+ delivered: int
+ # Handed to the queue; each row is removed when its job delivers and kept if it fails again.
+ enqueued: int
+ # Replays that failed again; their rows stay, with attempts raised by one.
+ failed: int
+ # Rows not replayed: the webhook was removed, disabled or unsubscribed, or a plugin cancelled it.
+ skipped: int
+ # Replayable rows still in scope after this call.
+ remaining: int
# ── Chat ──────────────────────────────────────────────────────────
diff --git a/sdk/python/tests/test_sdk.py b/sdk/python/tests/test_sdk.py
index b478a49b3..a4ffcb8d0 100644
--- a/sdk/python/tests/test_sdk.py
+++ b/sdk/python/tests/test_sdk.py
@@ -734,6 +734,16 @@ class TestWebhooks:
make_client(backend).webhooks.create("s", {"url": "u", "events": ["message.received"], "filters": filters})
assert backend.last_call.body == {"url": "u", "events": ["message.received"], "filters": filters}
+ def test_redrive_delivery_failures_posts_the_filter_or_an_empty_body(self):
+ result = {"redriven": 1, "delivered": 1, "enqueued": 0, "failed": 0, "skipped": 0, "remaining": 0}
+ backend = MockBackend().on("POST", "/api/webhooks/delivery-failures/redrive", body=result)
+ client = make_client(backend)
+ assert client.webhooks.redrive_delivery_failures({"sessionId": "s", "limit": 10}) == result
+ assert backend.last_call.method == "POST"
+ assert backend.last_call.body == {"sessionId": "s", "limit": 10}
+ client.webhooks.redrive_delivery_failures()
+ assert backend.last_call.body == {}
+
class TestStatus:
def test_send_image_video_forward_nested_media_body(self):
diff --git a/src/config/configuration.ts b/src/config/configuration.ts
index af43d20e9..8d0e3703d 100644
--- a/src/config/configuration.ts
+++ b/src/config/configuration.ts
@@ -310,6 +310,10 @@ export default () => ({
// 0 = don't wait (explicit opt-out); blank/garbage falls back to the default — a NaN here would
// silently remove the drain deadline downstream (Math.max(0, NaN) is NaN).
shutdownDrainMs: resolveNonNegativeIntEnv(process.env.WEBHOOK_SHUTDOWN_DRAIN_MS, 5000),
+ // How long a terminal delivery-failure row keeps the event data it was built from, so
+ // `POST /webhooks/delivery-failures/redrive` can replay it. 0 (the default) stores nothing, which
+ // is the pre-redrive behaviour. The retention sweep nulls an expired payload and keeps the row.
+ failurePayloadRetentionHours: resolveNonNegativeIntEnv(process.env.WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS, 0),
},
// API configuration
diff --git a/src/config/env-precedence.ts b/src/config/env-precedence.ts
index 754484f1b..342b9f6a1 100644
--- a/src/config/env-precedence.ts
+++ b/src/config/env-precedence.ts
@@ -168,6 +168,7 @@ export const BLANK_SHADOWED_ENV_KEYS: string[] = [
'WEBHOOK_MAX_PAYLOAD_BYTES',
'WEBHOOK_MAX_PER_SESSION',
'WEBHOOK_FAILURE_RETENTION_DAYS',
+ 'WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS',
'WEBHOOK_WORKER_CONCURRENCY',
'WEBHOOK_DEGRADED_SESSION_CONCURRENCY',
'INGRESS_INSTANCE_LIMIT',
diff --git a/src/config/env.validation.ts b/src/config/env.validation.ts
index f2689eedb..eeff25849 100644
--- a/src/config/env.validation.ts
+++ b/src/config/env.validation.ts
@@ -225,6 +225,7 @@ export function validateEnv(config: EnvConfig): EnvConfig {
'WEBHOOK_MAX_PER_SESSION', // 0 = unlimited
'AUTOMATION_MAX_PER_SESSION', // 0 = unlimited
'WEBHOOK_MEDIA_INLINE_MAX_BYTES', // 0 = never inline media
+ 'WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS', // 0 = failure rows keep no payload (no redrive)
'EXPORT_INLINE_MEDIA_BUDGET_BYTES', // 0 = a data export carries no inline media at all
'MESSAGE_LIST_INLINE_MEDIA_BUDGET_BYTES', // 0 = a message list carries no inline media at all
'CHAT_MEDIA_ARCHIVE_TTL_DAYS', // 0 = keep archived chat media forever
@@ -291,6 +292,13 @@ export function validateEnv(config: EnvConfig): EnvConfig {
errors.push(`${key} must be at most 36500 (got "${days}")`);
}
}
+ // The same cutoff bound, expressed in hours.
+ {
+ const hours = str('WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS');
+ if (hours !== undefined && Number(hours) > 36500 * 24) {
+ errors.push(`WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS must be at most ${36500 * 24} (got "${hours}")`);
+ }
+ }
// The grace windows build the same kind of cutoff (now minus the grace), so they share the cap.
for (const key of ['MESSAGE_REAPER_GRACE_MS', 'WEBHOOK_RECONCILE_GRACE_MS', 'INGRESS_RECONCILE_GRACE_MS']) {
const raw = str(key);
diff --git a/src/database/migrations/1787000000000-AddWebhookDeliveryFailurePayload.ts b/src/database/migrations/1787000000000-AddWebhookDeliveryFailurePayload.ts
new file mode 100644
index 000000000..b2c49595a
--- /dev/null
+++ b/src/database/migrations/1787000000000-AddWebhookDeliveryFailurePayload.ts
@@ -0,0 +1,23 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+/**
+ * Adds `webhook_delivery_failures.payload`: the event data a terminal delivery failure was built
+ * from, kept for WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS so `POST /webhooks/delivery-failures/redrive`
+ * can replay it. `text` on both dialects, the storage the entity's `simple-json` maps to (see
+ * jsonColumnType). NULL for every existing row, so nothing recorded before the upgrade is replayable,
+ * which keeps today's behaviour. Hand-authored because `synchronize` is off on the `data` connection
+ * for Postgres.
+ */
+export class AddWebhookDeliveryFailurePayload1787000000000 implements MigrationInterface {
+ name = 'AddWebhookDeliveryFailurePayload1787000000000';
+
+ public async up(queryRunner: QueryRunner): Promise<void> {
+ if (await queryRunner.hasColumn('webhook_delivery_failures', 'payload')) return;
+ await queryRunner.query(`ALTER TABLE "webhook_delivery_failures" ADD COLUMN "payload" text`);
+ }
+
+ public async down(queryRunner: QueryRunner): Promise<void> {
+ if (!(await queryRunner.hasColumn('webhook_delivery_failures', 'payload'))) return;
+ await queryRunner.query(`ALTER TABLE "webhook_delivery_failures" DROP COLUMN "payload"`);
+ }
+}
diff --git a/src/modules/audit/entities/audit-log.entity.ts b/src/modules/audit/entities/audit-log.entity.ts
index d2deff3ce..9edb97788 100644
--- a/src/modules/audit/entities/audit-log.entity.ts
+++ b/src/modules/audit/entities/audit-log.entity.ts
@@ -55,6 +55,7 @@ export enum AuditAction {
WEBHOOK_DELETED = 'webhook_deleted',
WEBHOOK_TRIGGERED = 'webhook_triggered',
WEBHOOK_FAILED = 'webhook_failed',
+ WEBHOOK_DELIVERIES_REDRIVEN = 'webhook_deliveries_redriven',
// Integration plugin-instance events
INTEGRATION_INSTANCE_CREATED = 'integration_instance_created',
diff --git a/src/modules/auth/global-route-fence-coverage.spec.ts b/src/modules/auth/global-route-fence-coverage.spec.ts
index 09a7ef501..b1dcd6ad5 100644
--- a/src/modules/auth/global-route-fence-coverage.spec.ts
+++ b/src/modules/auth/global-route-fence-coverage.spec.ts
@@ -20,6 +20,7 @@ const ALLOWLIST = new Map<string, string>([
['audit.controller.ts :: findAll', 'scope-filters rows via resolveSessionScope'],
['webhooks-list.controller.ts :: findAll', 'scope-filters rows via the calling key'],
['webhooks-list.controller.ts :: deliveryFailures', 'scope-filters rows via the calling key'],
+ ['webhooks-list.controller.ts :: redriveDeliveryFailures', 'replays only rows inside the calling key scope'],
['search.controller.ts :: search', 'scope-filters results via the calling key'],
['session.controller.ts :: findAll', 'scope-filters sessions to the key allowlist'],
['session.controller.ts :: getStats', 'scope-filters the aggregate to apiKey.allowedSessions in the service'],
diff --git a/src/modules/infra/export-tables.redaction.spec.ts b/src/modules/infra/export-tables.redaction.spec.ts
index 5151e920d..05331a14f 100644
--- a/src/modules/infra/export-tables.redaction.spec.ts
+++ b/src/modules/infra/export-tables.redaction.spec.ts
@@ -1,5 +1,5 @@
import { EXPORT_TABLES } from './export-tables';
-import type { SessionRow, WebhookRow } from './migration-tables.types';
+import type { SessionRow, WebhookDeliveryFailureRow, WebhookRow } from './migration-tables.types';
/**
* The credentials a backup payload must not carry.
@@ -92,4 +92,15 @@ describe('export redaction', () => {
expect(rows[0].url).toBe('https://example.com/hook');
});
});
+
+ describe('webhook delivery-failure replay payload', () => {
+ it('drops the replay copy of the event and keeps the record', () => {
+ const rows = runAfterRead('webhookDeliveryFailures', [
+ { id: 'f1', webhookId: 'w1', lastError: 'HTTP 503: x', payload: '{"body":"private"}' },
+ ] as unknown as WebhookDeliveryFailureRow[]);
+
+ expect(rows[0]).not.toHaveProperty('payload');
+ expect(rows[0].lastError).toBe('HTTP 503: x');
+ });
+ });
});
diff --git a/src/modules/infra/export-tables.ts b/src/modules/infra/export-tables.ts
index 42a0e55a9..81196ef6b 100644
--- a/src/modules/infra/export-tables.ts
+++ b/src/modules/infra/export-tables.ts
@@ -1,4 +1,11 @@
-import type { MigrationTables, SessionRow, WebhookRow, MessageRow, MessageBatchRow } from './migration-tables.types';
+import type {
+ MigrationTables,
+ SessionRow,
+ WebhookRow,
+ MessageRow,
+ MessageBatchRow,
+ WebhookDeliveryFailureRow,
+} from './migration-tables.types';
/**
* A `data` value that is a POINTER rather than bytes. `metadata.media.data` holds `base64 || dto.url!`
@@ -26,6 +33,17 @@ function redactWebhookCredentials(rows: WebhookRow[]): void {
}
}
+/**
+ * A delivery-failure row's `payload` is a short-lived replay copy of the event (a whole message body,
+ * cleared after WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS). It is not part of the record a backup keeps,
+ * and the importer never restores it, so it is left out of the archive rather than carried in it.
+ */
+function stripWebhookFailurePayload(rows: WebhookDeliveryFailureRow[]): void {
+ for (const row of rows) {
+ delete row.payload;
+ }
+}
+
/**
* `sessions.proxyUrl` may embed `user:pass` (docs/04 section 4.3), and the sibling webhook secret
* above is already kept out of this payload. Strip the userinfo and keep the rest of the URL.
@@ -280,7 +298,12 @@ export const EXPORT_TABLES: AnyExportTable[] = [
defineExportTable({ key: 'pluginInstances', table: 'plugin_instances', optional: true }),
defineExportTable({ key: 'conversationMappings', table: 'conversation_mappings', optional: true }),
defineExportTable({ key: 'ingressEvents', table: 'ingress_events', optional: true }),
- defineExportTable({ key: 'webhookDeliveryFailures', table: 'webhook_delivery_failures', optional: true }),
+ defineExportTable({
+ key: 'webhookDeliveryFailures',
+ table: 'webhook_delivery_failures',
+ optional: true,
+ afterRead: stripWebhookFailurePayload,
+ }),
defineExportTable({ key: 'webhookOutboxEvents', table: 'webhook_outbox_events', optional: true }),
defineExportTable({
key: 'integrationDeliveryFailures',
diff --git a/src/modules/infra/migration-tables.types.ts b/src/modules/infra/migration-tables.types.ts
index 932918838..99f9a9feb 100644
--- a/src/modules/infra/migration-tables.types.ts
+++ b/src/modules/infra/migration-tables.types.ts
@@ -184,6 +184,8 @@ export interface WebhookDeliveryFailureRow {
attempts: number;
lastStatusCode: number | null;
lastError: string;
+ /** Replay copy of the event; read by `SELECT *` but dropped from the export (see export-tables). */
+ payload?: string | null;
createdAt: string;
}
diff --git a/src/modules/queue/processors/webhook.processor.spec.ts b/src/modules/queue/processors/webhook.processor.spec.ts
index 6dff1ed23..6da78d869 100644
--- a/src/modules/queue/processors/webhook.processor.spec.ts
+++ b/src/modules/queue/processors/webhook.processor.spec.ts
@@ -187,6 +187,26 @@ describe('WebhookProcessor', () => {
);
});
+ it('keeps the job replay copy (pre-hook data) on the final-attempt row, never the sent body', async () => {
+ mockFetch.mockResolvedValue({ ok: false, status: 503, statusText: 'Service Unavailable' });
+ const job = makeJob({ maxRetries: 3, replayData: { from: '[email protected]', body: 'hi' } }, 2);
+ job.data.payload.data = { redacted: true };
+
+ await expect(processor.process(job)).rejects.toThrow();
+
+ expect(failureRepo.insert).toHaveBeenCalledWith(
+ expect.objectContaining({ attempts: 3, payload: { from: '[email protected]', body: 'hi' } }),
+ );
+ });
+
+ it('writes no payload for a job enqueued with payload retention off', async () => {
+ mockFetch.mockResolvedValue({ ok: false, status: 503, statusText: 'Service Unavailable' });
+
+ await expect(processor.process(makeJob({ maxRetries: 3 }, 2))).rejects.toThrow();
+
+ expect((failureRepo.insert.mock.calls as unknown[][])[0][0]).not.toHaveProperty('payload');
+ });
+
it("replaces a replayed delivery's attempts-0 shed row with its own final-attempt row", async () => {
// The shed row's dedup entry would otherwise suppress this row and keep only the capacity reason.
failureRows.push({ webhookId: 'wh-1', idempotencyKey: 'k', attempts: 0 });
diff --git a/src/modules/queue/processors/webhook.processor.ts b/src/modules/queue/processors/webhook.processor.ts
index f8537dee6..fb539a65c 100644
--- a/src/modules/queue/processors/webhook.processor.ts
+++ b/src/modules/queue/processors/webhook.processor.ts
@@ -322,6 +322,8 @@ export class WebhookProcessor extends WorkerHost {
attempts: job.attemptsMade + 1,
lastStatusCode: statusCodeFromError(errorMessage),
lastError: clientError,
+ // The producer attaches the pre-hook event data only while payload retention is on.
+ ...(job.data.replayData ? { payload: job.data.replayData } : {}),
});
if (recorded) {
incrementWebhookDeliveryFailures();
@@ -397,6 +399,7 @@ export class WebhookProcessor extends WorkerHost {
attempts: job.attemptsMade,
lastStatusCode: null, // no HTTP exchange completed on the stalled attempts
lastError: error.message,
+ ...(job.data.replayData ? { payload: job.data.replayData } : {}),
});
if (recorded) {
incrementWebhookDeliveryFailures();
diff --git a/src/modules/webhook/dto/webhook.dto.ts b/src/modules/webhook/dto/webhook.dto.ts
index 06a36893d..78b2e7e03 100644
--- a/src/modules/webhook/dto/webhook.dto.ts
+++ b/src/modules/webhook/dto/webhook.dto.ts
@@ -1,5 +1,6 @@
import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger';
import {
+ ArrayMaxSize,
ArrayMinSize,
IsArray,
IsBoolean,
@@ -9,6 +10,7 @@ import {
IsString,
IsUrl,
Max,
+ MaxLength,
Min,
MinLength,
ValidateIf,
@@ -415,6 +417,79 @@ export class WebhookDeliveryFailureDto {
@ApiProperty({ type: String, format: 'date-time', description: 'When the failure was first recorded.' })
createdAt!: Date;
+
+ @ApiProperty({
+ description:
+ 'True when the row still holds the event data and can be replayed with ' +
+ '`POST /webhooks/delivery-failures/redrive`. Only terminal rows (attempts > 0) recorded while ' +
+ '`WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS` > 0 are, until that window passes.',
+ example: false,
+ })
+ replayable!: boolean;
+}
+
+/** Body of `POST /webhooks/delivery-failures/redrive`. Every field narrows; an empty body takes the oldest rows. */
+export class RedriveWebhookDeliveriesDto {
+ @ApiPropertyOptional({
+ description: "Only rows of this session. Narrows within the calling key's allowedSessions, never past it.",
+ example: 'my-session',
+ })
+ @IsOptional()
+ @IsString()
+ @MaxLength(128)
+ sessionId?: string;
+
+ @ApiPropertyOptional({ description: 'Only rows of this webhook.', example: '0a941dac-a965-45e7-b318-74ae8be134f0' })
+ @IsOptional()
+ @IsString()
+ @MaxLength(128)
+ webhookId?: string;
+
+ @ApiPropertyOptional({
+ type: [String],
+ description: 'Only these failure rows (ids from `GET /webhooks/delivery-failures`).',
+ example: ['0a941dac-a965-45e7-b318-74ae8be134f0'],
+ })
+ @IsOptional()
+ @IsArray()
+ @ArrayMaxSize(500)
+ @IsString({ each: true })
+ @MaxLength(128, { each: true })
+ ids?: string[];
+
+ @ApiPropertyOptional({ description: 'Max rows replayed by this call (1-500, default 100).', example: 100 })
+ @IsOptional()
+ @IsInt()
+ @Min(1)
+ @Max(500)
+ limit?: number;
+}
+
+/** Outcome of `POST /webhooks/delivery-failures/redrive`. */
+export class WebhookRedriveResultDto {
+ @ApiProperty({ description: 'Rows replayed by this call: delivered plus enqueued.', example: 3 })
+ redriven!: number;
+
+ @ApiProperty({ description: 'Delivered by a direct POST; their failure rows were removed.', example: 2 })
+ delivered!: number;
+
+ @ApiProperty({
+ description: 'Handed to the queue; each row is removed when its job delivers and kept if it fails again.',
+ example: 1,
+ })
+ enqueued!: number;
+
+ @ApiProperty({ description: 'Replays that failed again; their rows stay, with attempts raised by one.', example: 0 })
+ failed!: number;
+
+ @ApiProperty({
+ description: 'Rows not replayed: the webhook was removed, disabled or unsubscribed, or a plugin cancelled it.',
+ example: 0,
+ })
+ skipped!: number;
+
+ @ApiProperty({ description: 'Replayable rows still in scope after this call.', example: 0 })
+ remaining!: number;
}
/** Outcome of `POST /sessions/:sessionId/webhooks/:id/test`. */
diff --git a/src/modules/webhook/entities/webhook-delivery-failure.entity.ts b/src/modules/webhook/entities/webhook-delivery-failure.entity.ts
index f0c8b8d37..8cf658a0e 100644
--- a/src/modules/webhook/entities/webhook-delivery-failure.entity.ts
+++ b/src/modules/webhook/entities/webhook-delivery-failure.entity.ts
@@ -1,5 +1,6 @@
import { Entity, PrimaryGeneratedColumn, Column, CreateDateColumn, Index } from 'typeorm';
import { NulFreeTransformer } from '../../../common/transformers/nul-free.transformer';
+import { jsonColumnType } from '../../../common/utils/column-types';
/**
* A durable record of a webhook delivery that exhausted all of its retries. The queued path (BullMQ)
@@ -50,6 +51,19 @@ export class WebhookDeliveryFailure {
@Column({ type: 'text', transformer: NulFreeTransformer })
lastError!: string;
+ /**
+ * The event data the delivery was built from (pre-`webhook:before`, after inline-media shedding),
+ * kept so `POST /webhooks/delivery-failures/redrive` can replay the lost event. Written only for a
+ * terminal row (attempts > 0) and only while WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS > 0; the
+ * retention sweep nulls it once that window passes, leaving the row itself for the audit trail.
+ * NULL on every row recorded with the knob off, which keeps today's behaviour.
+ *
+ * `select: false`: the column can hold a whole message body, and the list endpoint serializes
+ * these entities as they are read. Only the redrive path selects it explicitly.
+ */
+ @Column({ type: jsonColumnType(), nullable: true, select: false })
+ payload!: Record<string, unknown> | null;
+
/** When the delivery was finally abandoned. */
@CreateDateColumn()
createdAt!: Date;
diff --git a/src/modules/webhook/utils/record-delivery-failure.spec.ts b/src/modules/webhook/utils/record-delivery-failure.spec.ts
index add500a87..12f56a161 100644
--- a/src/modules/webhook/utils/record-delivery-failure.spec.ts
+++ b/src/modules/webhook/utils/record-delivery-failure.spec.ts
@@ -221,6 +221,36 @@ describe('recordWebhookDeliveryFailure', () => {
await expect(recordWebhookDeliveryFailure(repoWith(insert), logger, input)).resolves.toBe(true);
expect(logger.error).toHaveBeenCalled();
});
+
+ it('keeps the replay payload on a terminal row', async () => {
+ const insert = jest.fn().mockResolvedValue({});
+ const payload = { id: 'msg-1', body: 'hi' };
+
+ await recordWebhookDeliveryFailure(repoWith(insert), { error: jest.fn() }, { ...input, payload });
+
+ expect(insert).toHaveBeenCalledWith(expect.objectContaining({ attempts: 3, payload }));
+ });
+
+ it('never stores a payload on an attempts-0 row: its pending outbox row already holds the data', async () => {
+ const insert = jest.fn().mockResolvedValue({});
+
+ await recordWebhookDeliveryFailure(
+ repoWith(insert),
+ { error: jest.fn() },
+ { ...input, attempts: 0, payload: { id: 'msg-1' } },
+ );
+
+ expect(insert).toHaveBeenCalledTimes(1);
+ expect((insert.mock.calls as unknown[][])[0][0]).not.toHaveProperty('payload');
+ });
+
+ it('writes no payload column at all when none is given (payload retention off)', async () => {
+ const insert = jest.fn().mockResolvedValue({});
+
+ await recordWebhookDeliveryFailure(repoWith(insert), { error: jest.fn() }, input);
+
+ expect((insert.mock.calls as unknown[][])[0][0]).not.toHaveProperty('payload');
+ });
});
describe('clearDeliveryFailureRows', () => {
diff --git a/src/modules/webhook/utils/record-delivery-failure.ts b/src/modules/webhook/utils/record-delivery-failure.ts
index 662e1a159..0591feb08 100644
--- a/src/modules/webhook/utils/record-delivery-failure.ts
+++ b/src/modules/webhook/utils/record-delivery-failure.ts
@@ -1,4 +1,5 @@
import { MoreThan, Repository } from 'typeorm';
+import type { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity';
import { WebhookDeliveryFailure } from '../entities/webhook-delivery-failure.entity';
export interface WebhookDeliveryFailureInput {
@@ -11,6 +12,12 @@ export interface WebhookDeliveryFailureInput {
attempts: number;
lastStatusCode?: number | null;
lastError: string;
+ /**
+ * The pre-hook event data to keep for a later redrive. Passed only while
+ * WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS > 0, and persisted only on a terminal row: an attempts-0
+ * row keeps its outbox row pending, which already holds the same data for the reconciler.
+ */
+ payload?: Record<string, unknown> | null;
}
/** Minimal logger shape — both WebhookProcessor and WebhookService pass their `createLogger` instance. */
@@ -112,7 +119,14 @@ export async function recordWebhookDeliveryFailure(
return false;
}
}
- await repo.insert({ ...input, lastStatusCode: input.lastStatusCode ?? null });
+ const { payload, ...row } = input;
+ // The cast covers `payload` only: TypeORM's insert typing cannot see through a free-form JSON
+ // record, while simple-json serializes it like the outbox payload.
+ await repo.insert({
+ ...row,
+ lastStatusCode: input.lastStatusCode ?? null,
+ ...(terminal && payload ? { payload } : {}),
+ } as QueryDeepPartialEntity<WebhookDeliveryFailure>);
if (terminal) {
// Only after the insert, so a crash in between leaves both rows, never none. A later terminal
// record of the same delivery repeats this clear.
diff --git a/src/modules/webhook/webhook-delivery.service.spec.ts b/src/modules/webhook/webhook-delivery.service.spec.ts
index 920eb94ce..631f01a06 100644
--- a/src/modules/webhook/webhook-delivery.service.spec.ts
+++ b/src/modules/webhook/webhook-delivery.service.spec.ts
@@ -1626,6 +1626,23 @@ describe('WebhookDeliveryService', () => {
// Custom headers may carry receiver credentials and the processor rebuilds every header from
// the current row, so the job must not copy them into Redis.
expect(jobData).not.toHaveProperty('headers');
+ // Payload retention is off by default, so the job carries no second copy of the event.
+ expect(jobData).not.toHaveProperty('replayData');
+ });
+
+ it('carries the pre-hook event data in the job only while payload retention is on', async () => {
+ const queueService = await buildQueueService((key: string, def?: unknown) => {
+ if (key === 'queue.enabled') return true;
+ if (key === 'webhook.failurePayloadRetentionHours') return 72;
+ return def;
+ });
+ (repository.find as jest.Mock).mockResolvedValue([createMockWebhook({ events: ['message.received'] })]);
+ (hookManager.execute as jest.Mock).mockResolvedValue({ continue: true, data: {} });
+
+ await queueService.dispatch('sess-1', 'message.received', { from: '[email protected]' });
+
+ const [, jobData] = (webhookQueue.add as jest.Mock).mock.calls[0] as [string, WebhookJobData];
+ expect(jobData.replayData).toEqual({ from: '[email protected]' });
});
it('leaves the shed row of a replay in place when its job is queued', async () => {
@@ -1964,6 +1981,58 @@ describe('WebhookDeliveryService', () => {
expect((sleep as unknown as jest.Mock).mock.calls.map(([ms]) => ms as number)).toEqual([100, 200, 400]);
expect(mockFetch).toHaveBeenCalledTimes(4);
});
+
+ it('makes exactly one attempt for an operator redrive (singleAttempt), with no backoff', async () => {
+ const webhook = createMockWebhook({ retryCount: 4 });
+ mockFetch.mockRejectedValue(new Error('receiver down'));
+ (sleep as unknown as jest.Mock).mockClear();
+
+ await expect(
+ service.redeliver(webhook, 'sess-1', 'message.received', 'k', {}, { singleAttempt: true }),
+ ).resolves.toBe('failed');
+
+ expect(mockFetch).toHaveBeenCalledTimes(1);
+ expect(sleep).not.toHaveBeenCalled();
+ expect(insertedFailures).toEqual([expect.objectContaining({ attempts: 1, idempotencyKey: 'k' })]);
+ });
+
+ it('keeps no replay payload on the terminal row while payload retention is off (the default)', async () => {
+ const webhook = createMockWebhook({ retryCount: 1 });
+ mockFetch.mockRejectedValue(new Error('receiver down'));
+
+ await service.redeliver(webhook, 'sess-1', 'message.received', 'k', { from: '[email protected]' });
+
+ expect(insertedFailures).toHaveLength(1);
+ expect(insertedFailures[0]).not.toHaveProperty('payload');
+ });
+
+ it('keeps the PRE-hook event data on the terminal row while payload retention is on', async () => {
+ (configService.get as jest.Mock).mockImplementation(<T>(key: string, def?: T) => {
+ if (key === 'webhook.failurePayloadRetentionHours') return 24;
+ if (key === 'webhook.retryDelay') return 100;
+ return def;
+ });
+ // A hook that rewrites the body: the stored copy must be the data before it, since a redrive
+ // runs the hooks again.
+ (hookManager.execute as jest.Mock).mockImplementation((name: string, ctx: { payload?: WebhookPayload }) =>
+ Promise.resolve(
+ name === 'webhook:before' && ctx.payload
+ ? { continue: true, data: { payload: { ...ctx.payload, data: { redacted: true } } } }
+ : { continue: true, data: {} },
+ ),
+ );
+ const webhook = createMockWebhook({ retryCount: 1 });
+ mockFetch.mockRejectedValue(new Error('receiver down'));
+
+ await service.redeliver(webhook, 'sess-1', 'message.received', 'k', { from: '[email protected]', body: 'hi' });
+
+ expect(JSON.parse((mockFetch.mock.calls[0] as [string, SentInit])[1].body)).toMatchObject({
+ data: { redacted: true },
+ });
+ expect(insertedFailures).toEqual([
+ expect.objectContaining({ attempts: 1, payload: { from: '[email protected]', body: 'hi' } }),
+ ]);
+ });
});
describe('dispatch slot isolation (direct mode)', () => {
diff --git a/src/modules/webhook/webhook-delivery.service.ts b/src/modules/webhook/webhook-delivery.service.ts
index 5e66cc5ac..94ad5af73 100644
--- a/src/modules/webhook/webhook-delivery.service.ts
+++ b/src/modules/webhook/webhook-delivery.service.ts
@@ -43,6 +43,13 @@ export interface WebhookJobData {
payload: WebhookPayload;
attempt: number;
maxRetries: number;
+ /**
+ * The pre-`webhook:before` event data, carried only while WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS
+ * > 0 so the processor's terminal failure row can keep it for a redrive. `payload.data` is the
+ * post-hook body and must not be replayed: the redrive runs the hooks again. Bounded like `payload`
+ * (inline media over WEBHOOK_MEDIA_INLINE_MAX_BYTES was shed before it was captured).
+ */
+ replayData?: Record<string, unknown>;
}
/**
@@ -81,6 +88,19 @@ interface DispatchEventContext {
baseData: Record<string, unknown>;
/** Gives the dispatch slot up while `fn` runs (a retry backoff). Absent outside the limiter. */
yieldSlot?: <R>(fn: () => Promise<R>) => Promise<R>;
+ /**
+ * Direct delivery makes one attempt instead of `webhook.retryCount`. Set by an operator redrive,
+ * which runs inside an HTTP request: the in-process backoff schedule would hold it for minutes.
+ */
+ singleAttempt?: boolean;
+}
+
+/** Per-delivery options of the direct path (deliverWebhook), derived from the dispatch context. */
+interface DirectDeliveryOptions {
+ yieldSlot?: <R>(fn: () => Promise<R>) => Promise<R>;
+ /** Kept on the terminal failure row for a redrive; set only while payload retention is on. */
+ replayData?: Record<string, unknown>;
+ singleAttempt?: boolean;
}
/** The limiter refused or dropped the task because shutdown closed it. */
@@ -545,6 +565,7 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
payload: finalPayload,
attempt: 1,
maxRetries: webhook.retryCount,
+ ...(this.keepsFailurePayload() ? { replayData: ctx.baseData } : {}),
};
await this.webhookQueue!.add(`webhook-${webhook.id}`, jobData, {
@@ -595,7 +616,7 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
// same X-OpenWA-Idempotency-Key / X-OpenWA-Delivery-Id, so a conformant receiver dedupes.
try {
// Removed, disabled or unsubscribed before a retry: nothing to report, as on the queued path.
- if (!(await this.deliverWebhook(webhook, finalPayload, body, ctx.yieldSlot))) return 'cancelled';
+ if (!(await this.deliverWebhook(webhook, finalPayload, body, this.directOptions(ctx)))) return 'cancelled';
await this.hookManager.execute(
'webhook:delivered',
@@ -647,7 +668,7 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
const { sessionId, event } = ctx;
try {
// Removed, disabled or unsubscribed before a retry: nothing to report, as on the queued path.
- if (!(await this.deliverWebhook(webhook, finalPayload, body, ctx.yieldSlot))) return 'cancelled';
+ if (!(await this.deliverWebhook(webhook, finalPayload, body, this.directOptions(ctx)))) return 'cancelled';
// Execute hook after successful delivery
await this.hookManager.execute(
@@ -893,9 +914,32 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
event: string,
idempotencyKey: string,
data: Record<string, unknown>,
+ options: { singleAttempt?: boolean } = {},
): Promise<WebhookDeliveryOutcome> {
const deliveryId = generateDeliveryId();
- return this.deliverOne(webhook, deliveryId, idempotencyKey, { sessionId, event, baseData: data });
+ return this.deliverOne(webhook, deliveryId, idempotencyKey, {
+ sessionId,
+ event,
+ baseData: data,
+ singleAttempt: options.singleAttempt,
+ });
+ }
+
+ /**
+ * WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS > 0: a terminal failure row keeps the event data it was
+ * built from, so an operator can redrive it. Off (0) by default, which stores nothing new.
+ */
+ private keepsFailurePayload(): boolean {
+ return this.configService.get<number>('webhook.failurePayloadRetentionHours', 0) > 0;
+ }
+
+ /** What the direct path needs from the dispatch context: the slot, the replay copy, the attempt cap. */
+ private directOptions(ctx: DispatchEventContext): DirectDeliveryOptions {
+ return {
+ yieldSlot: ctx.yieldSlot,
+ replayData: this.keepsFailurePayload() ? ctx.baseData : undefined,
+ singleAttempt: ctx.singleAttempt,
+ };
}
/**
@@ -913,8 +957,9 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
webhook: Webhook,
payload: WebhookPayload,
body: string,
- yieldSlot: <R>(fn: () => Promise<R>) => Promise<R> = fn => fn(),
+ options: DirectDeliveryOptions = {},
): Promise<boolean> {
+ const yieldSlot = options.yieldSlot ?? (<R>(fn: () => Promise<R>): Promise<R> => fn());
const delay = this.configService.get<number>('webhook.retryDelay', 5000);
let current = webhook;
for (let attempt = 1; ; attempt++) {
@@ -980,7 +1025,7 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
action: 'webhook_delivery_failed',
});
- if (attempt < current.retryCount) {
+ if (attempt < (options.singleAttempt ? 1 : current.retryCount)) {
// Without the dispatch slot: a backoff sends nothing, and holding the slot through it let
// a few failing receivers stall every other delivery. Taking it back throws once shutdown
// closed the limiter, which skips the terminal record below: the event was not given up.
@@ -1000,6 +1045,7 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
deliveryId: payload.deliveryId,
attempts: attempt,
error,
+ ...(options.replayData ? { payload: options.replayData } : {}),
});
if (recorded) {
incrementWebhookDeliveryFailures();
diff --git a/src/modules/webhook/webhook-redrive.service.spec.ts b/src/modules/webhook/webhook-redrive.service.spec.ts
new file mode 100644
index 000000000..ae7181f10
--- /dev/null
+++ b/src/modules/webhook/webhook-redrive.service.spec.ts
@@ -0,0 +1,196 @@
+import { FindOperator, Repository } from 'typeorm';
+import { Webhook } from './entities/webhook.entity';
+import { WebhookDeliveryFailure } from './entities/webhook-delivery-failure.entity';
+import { WebhookDeliveryService } from './webhook-delivery.service';
+import { MAX_WEBHOOK_REDRIVE_LIMIT, WebhookRedriveService } from './webhook-redrive.service';
+
+const failure = (overrides: Partial<WebhookDeliveryFailure> = {}): WebhookDeliveryFailure => ({
+ id: 'f-1',
+ webhookId: 'wh-1',
+ sessionId: 'sess-1',
+ event: 'message.received',
+ url: 'https://r.example/h',
+ idempotencyKey: 'key-1',
+ deliveryId: 'd-1',
+ attempts: 3,
+ lastStatusCode: 503,
+ lastError: 'HTTP 503: x',
+ payload: { id: 'msg-1', body: 'hi' },
+ createdAt: new Date('2026-10-01T00:00:00Z'),
+ ...overrides,
+});
+
+const webhook = (overrides: Partial<Webhook> = {}): Webhook =>
+ ({
+ id: 'wh-1',
+ sessionId: 'sess-1',
+ url: 'https://r.example/h',
+ events: ['message.received'],
+ active: true,
+ retryCount: 3,
+ ...overrides,
+ }) as Webhook;
+
+describe('WebhookRedriveService', () => {
+ let failures: { find: jest.Mock; count: jest.Mock; update: jest.Mock };
+ let webhooks: { find: jest.Mock };
+ let delivery: { redeliver: jest.Mock };
+ let service: WebhookRedriveService;
+
+ beforeEach(() => {
+ failures = {
+ find: jest.fn().mockResolvedValue([failure()]),
+ count: jest.fn().mockResolvedValue(0),
+ update: jest.fn().mockResolvedValue({ affected: 1 }),
+ };
+ webhooks = { find: jest.fn().mockResolvedValue([webhook()]) };
+ delivery = { redeliver: jest.fn().mockResolvedValue('delivered') };
+ service = new WebhookRedriveService(
+ webhooks as unknown as Repository<Webhook>,
+ failures as unknown as Repository<WebhookDeliveryFailure>,
+ delivery as unknown as WebhookDeliveryService,
+ );
+ });
+
+ const findArgs = (): {
+ select: Record<string, boolean>;
+ where: Record<string, unknown>;
+ take: number;
+ order: Record<string, string>;
+ } => (failures.find.mock.calls as unknown[][])[0][0] as ReturnType<typeof findArgs>;
+
+ it('replays a stored row with its STORED idempotency key, one attempt, and reports it delivered', async () => {
+ const result = await service.redrive({});
+
+ expect(delivery.redeliver).toHaveBeenCalledWith(
+ expect.objectContaining({ id: 'wh-1' }),
+ 'sess-1',
+ 'message.received',
+ 'key-1',
+ { id: 'msg-1', body: 'hi' },
+ { singleAttempt: true },
+ );
+ expect(result).toEqual({ redriven: 1, delivered: 1, enqueued: 0, failed: 0, skipped: 0, remaining: 0 });
+ });
+
+ it('reads the select:false payload explicitly, takes only terminal keyed rows, oldest first', async () => {
+ await service.redrive({});
+
+ const args = findArgs();
+ expect(args.select.payload).toBe(true);
+ expect(args.where.attempts).toBeInstanceOf(FindOperator);
+ expect((args.where.attempts as FindOperator<number>).type).toBe('moreThan');
+ expect((args.where.payload as FindOperator<unknown>).type).toBe('not');
+ expect((args.where.idempotencyKey as FindOperator<unknown>).type).toBe('not');
+ expect(args.order).toEqual({ createdAt: 'ASC', id: 'ASC' });
+ expect(args.take).toBe(100);
+ });
+
+ it('clamps the limit to the maximum batch', async () => {
+ await service.redrive({ limit: 10_000 });
+ expect(findArgs().take).toBe(MAX_WEBHOOK_REDRIVE_LIMIT);
+ });
+
+ it('confines a scoped key to its allowedSessions, and a sessionId outside them to nothing', async () => {
+ await service.redrive({}, ['sess-1', 'sess-2']);
+ expect((findArgs().where.sessionId as FindOperator<string[]>).value).toEqual(['sess-1', 'sess-2']);
+
+ failures.find.mockClear();
+ const result = await service.redrive({ sessionId: 'sess-9' }, ['sess-1']);
+ expect(failures.find).not.toHaveBeenCalled();
+ expect(delivery.redeliver).toHaveBeenCalledTimes(1);
+ expect(result.redriven).toBe(0);
+ });
+
+ it('narrows by webhookId and explicit ids', async () => {
+ await service.redrive({ webhookId: 'wh-1', ids: ['f-1', 'f-2'] });
+ const where = findArgs().where;
+ expect(where.webhookId).toBe('wh-1');
+ expect((where.id as FindOperator<string[]>).value).toEqual(['f-1', 'f-2']);
+ });
+
+ it('skips a row whose webhook is gone, disabled, unsubscribed or now owned by another session', async () => {
+ failures.find.mockResolvedValue([
+ failure({ id: 'a', webhookId: 'gone' }),
+ failure({ id: 'b', webhookId: 'off' }),
+ failure({ id: 'c', webhookId: 'unsub' }),
+ failure({ id: 'd', webhookId: 'moved' }),
+ ]);
+ webhooks.find.mockResolvedValue([
+ webhook({ id: 'off', active: false }),
+ webhook({ id: 'unsub', events: ['message.ack'] }),
+ webhook({ id: 'moved', sessionId: 'sess-2' }),
+ ]);
+
+ const result = await service.redrive({});
+
+ expect(delivery.redeliver).not.toHaveBeenCalled();
+ expect(result.skipped).toBe(4);
+ });
+
+ it('counts enqueued and cancelled outcomes, and raises attempts on a replay that fails again', async () => {
+ failures.find.mockResolvedValue([
+ failure({ id: 'a', attempts: 3 }),
+ failure({ id: 'b', attempts: 5 }),
+ failure({ id: 'c' }),
+ failure({ id: 'd', attempts: 2 }),
+ ]);
+ delivery.redeliver
+ .mockResolvedValueOnce('enqueued')
+ .mockResolvedValueOnce('failed')
+ .mockResolvedValueOnce('cancelled')
+ .mockRejectedValueOnce(new Error('boom'));
+ failures.count.mockResolvedValue(2);
+
+ const result = await service.redrive({});
+
+ expect(result).toEqual({ redriven: 1, delivered: 0, enqueued: 1, failed: 2, skipped: 1, remaining: 2 });
+ expect(failures.update).toHaveBeenCalledWith({ id: 'b' }, { attempts: 6 });
+ expect(failures.update).toHaveBeenCalledWith({ id: 'd' }, { attempts: 3 });
+ expect(failures.update).toHaveBeenCalledTimes(2);
+ });
+
+ it('keeps the batch going when recording a failed replay fails', async () => {
+ failures.find.mockResolvedValue([failure({ id: 'a' }), failure({ id: 'b' })]);
+ delivery.redeliver.mockResolvedValueOnce('failed').mockResolvedValueOnce('delivered');
+ failures.update.mockRejectedValue(new Error('db down'));
+
+ const result = await service.redrive({});
+
+ expect(result.failed).toBe(1);
+ expect(result.delivered).toBe(1);
+ });
+
+ it('runs overlapping calls one after the other, and survives a failed batch', async () => {
+ let release!: () => void;
+ const gate = new Promise<void>(resolve => (release = resolve));
+ delivery.redeliver.mockImplementationOnce(async () => {
+ await gate;
+ return 'delivered';
+ });
+ failures.find.mockRejectedValueOnce(new Error('db down')).mockResolvedValue([failure()]);
+
+ const first = service.redrive({});
+ await expect(first).rejects.toThrow('db down');
+
+ const second = service.redrive({});
+ const third = service.redrive({});
+ await new Promise(resolve => setImmediate(resolve));
+ // The third call has not read anything while the second is still replaying.
+ expect(failures.find).toHaveBeenCalledTimes(2);
+ release();
+ await expect(second).resolves.toMatchObject({ delivered: 1 });
+ await expect(third).resolves.toMatchObject({ delivered: 1 });
+ expect(failures.find).toHaveBeenCalledTimes(3);
+ });
+
+ it('does not read webhooks when nothing is replayable', async () => {
+ failures.find.mockResolvedValue([]);
+ failures.count.mockResolvedValue(0);
+
+ const result = await service.redrive({});
+
+ expect(webhooks.find).not.toHaveBeenCalled();
+ expect(result).toEqual({ redriven: 0, delivered: 0, enqueued: 0, failed: 0, skipped: 0, remaining: 0 });
+ });
+});
diff --git a/src/modules/webhook/webhook-redrive.service.ts b/src/modules/webhook/webhook-redrive.service.ts
new file mode 100644
index 000000000..d784057cc
--- /dev/null
+++ b/src/modules/webhook/webhook-redrive.service.ts
@@ -0,0 +1,197 @@
+import { Injectable } from '@nestjs/common';
+import { InjectRepository } from '@nestjs/typeorm';
+import { FindOptionsWhere, In, IsNull, MoreThan, Not, Repository } from 'typeorm';
+import { Webhook } from './entities/webhook.entity';
+import { WebhookDeliveryFailure } from './entities/webhook-delivery-failure.entity';
+import { WebhookDeliveryService } from './webhook-delivery.service';
+import { isDeliverableWebhook } from './utils/deliver-once';
+import { resolveSessionScope } from '../../common/security/session-scope';
+import { createLogger } from '../../common/services/logger.service';
+
+/** Rows replayed per call when the caller names no limit, and the most one call will take. */
+export const DEFAULT_WEBHOOK_REDRIVE_LIMIT = 100;
+export const MAX_WEBHOOK_REDRIVE_LIMIT = 500;
+
+/**
+ * Replays run this many at a time. Each direct replay is one POST bounded by WEBHOOK_TIMEOUT, so a
+ * full batch against a receiver that times out finishes in about limit / 4 timeouts rather than
+ * limit of them, without opening a socket per row.
+ */
+const REDRIVE_CONCURRENCY = 4;
+
+export interface WebhookRedriveRequest {
+ sessionId?: string;
+ webhookId?: string;
+ ids?: string[];
+ limit?: number;
+}
+
+export interface WebhookRedriveResult {
+ /** Rows replayed by this call: delivered plus enqueued. */
+ redriven: number;
+ /** Delivered by a direct POST; their failure rows are gone. */
+ delivered: number;
+ /** Handed to the queue; the row is removed when the job delivers, and kept when it fails again. */
+ enqueued: number;
+ /** The replay failed again; the row stays, with its attempt count raised. */
+ failed: number;
+ /** Not replayed: the webhook was removed, disabled or unsubscribed, or a plugin cancelled it. */
+ skipped: number;
+ /** Replayable rows still in scope after this call. */
+ remaining: number;
+}
+
+/**
+ * Operator redrive of lost outbound webhook deliveries: the outbound twin of the integration
+ * RedriveService. A terminal row of webhook_delivery_failures keeps the pre-hook event data while
+ * WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS > 0; this replays those rows through
+ * WebhookDeliveryService.redeliver with the STORED idempotency key, so a receiver that already
+ * processed the event (the POST timed out after it was handled) dedups the replay instead of
+ * acting twice. `webhook:before` hooks run again, as on a reconciler replay.
+ *
+ * A replay that succeeds removes its own row (the delivery path clears every failure row of the
+ * key). One that fails again keeps the row: the recorder files one row per lost delivery, so the
+ * replay only raises its attempt count.
+ */
+@Injectable()
+export class WebhookRedriveService {
+ private readonly logger = createLogger('WebhookRedrive');
+ /**
+ * One redrive at a time on this node, so two overlapping calls (a double click, a retrying script)
+ * do not replay the same rows side by side. Across nodes a duplicate replay carries the same
+ * idempotency key, which the receiver dedups on.
+ */
+ private running: Promise<unknown> = Promise.resolve();
+
+ constructor(
+ @InjectRepository(Webhook, 'data')
+ private readonly webhookRepository: Repository<Webhook>,
+ @InjectRepository(WebhookDeliveryFailure, 'data')
+ private readonly failureRepository: Repository<WebhookDeliveryFailure>,
+ private readonly delivery: WebhookDeliveryService,
+ ) {}
+
+ redrive(request: WebhookRedriveRequest, allowedSessions?: string[] | null): Promise<WebhookRedriveResult> {
+ const run = this.running.then(() => this.redriveBatch(request, allowedSessions));
+ // The chain must survive a failed batch, or every later call would inherit its rejection.
+ this.running = run.catch(() => undefined);
+ return run;
+ }
+
+ private async redriveBatch(
+ request: WebhookRedriveRequest,
+ allowedSessions?: string[] | null,
+ ): Promise<WebhookRedriveResult> {
+ const result: WebhookRedriveResult = {
+ redriven: 0,
+ delivered: 0,
+ enqueued: 0,
+ failed: 0,
+ skipped: 0,
+ remaining: 0,
+ };
+ // The calling key's allowedSessions is authoritative; the body's sessionId may only narrow it.
+ const sessionScope = resolveSessionScope(allowedSessions, request.sessionId);
+ if (sessionScope !== null && sessionScope.length === 0) return result;
+
+ // Replayable: terminal (an attempts-0 row is the outbox's to replay), with a stored payload and
+ // the key the receiver dedups on.
+ const where: FindOptionsWhere<WebhookDeliveryFailure> = {
+ attempts: MoreThan(0),
+ payload: Not(IsNull()),
+ idempotencyKey: Not(IsNull()),
+ ...(sessionScope ? { sessionId: In(sessionScope) } : {}),
+ ...(request.webhookId ? { webhookId: request.webhookId } : {}),
+ };
+ const limit = Math.min(Math.max(1, request.limit ?? DEFAULT_WEBHOOK_REDRIVE_LIMIT), MAX_WEBHOOK_REDRIVE_LIMIT);
+ const rows = await this.failureRepository.find({
+ // `payload` is select: false on the entity; name every column so it is read here.
+ select: {
+ id: true,
+ webhookId: true,
+ sessionId: true,
+ event: true,
+ url: true,
+ idempotencyKey: true,
+ deliveryId: true,
+ attempts: true,
+ lastStatusCode: true,
+ lastError: true,
+ payload: true,
+ createdAt: true,
+ },
+ where: request.ids?.length ? { ...where, id: In(request.ids) } : where,
+ // Oldest first: the order the events were lost in.
+ order: { createdAt: 'ASC', id: 'ASC' },
+ take: limit,
+ });
+
+ if (rows.length > 0) {
+ const webhookIds = [...new Set(rows.map(r => r.webhookId))];
+ const webhooks = new Map(
+ (await this.webhookRepository.find({ where: { id: In(webhookIds) } })).map(w => [w.id, w] as const),
+ );
+ const queue = [...rows];
+ const worker = async (): Promise<void> => {
+ for (let row = queue.shift(); row; row = queue.shift()) {
+ await this.redriveRow(row, webhooks.get(row.webhookId) ?? null, result);
+ }
+ };
+ await Promise.all(Array.from({ length: Math.min(REDRIVE_CONCURRENCY, rows.length) }, worker));
+ }
+
+ result.redriven = result.delivered + result.enqueued;
+ result.remaining = await this.failureRepository.count({ where });
+ return result;
+ }
+
+ private async redriveRow(
+ row: WebhookDeliveryFailure,
+ webhook: Webhook | null,
+ result: WebhookRedriveResult,
+ ): Promise<void> {
+ // The row's own session must still own the webhook: a row is never replayed into another session.
+ if (!webhook || webhook.sessionId !== row.sessionId || !isDeliverableWebhook(webhook, row.event)) {
+ result.skipped++;
+ return;
+ }
+ try {
+ const outcome = await this.delivery.redeliver(
+ webhook,
+ row.sessionId,
+ row.event,
+ row.idempotencyKey,
+ row.payload ?? {},
+ { singleAttempt: true },
+ );
+ if (outcome === 'delivered') result.delivered++;
+ else if (outcome === 'enqueued') result.enqueued++;
+ else if (outcome === 'cancelled') result.skipped++;
+ else {
+ result.failed++;
+ await this.bumpAttempts(row);
+ }
+ } catch (error) {
+ result.failed++;
+ this.logger.error('Webhook redrive failed', error instanceof Error ? error.message : String(error), {
+ webhookId: row.webhookId,
+ failureId: row.id,
+ action: 'webhook_redrive_error',
+ });
+ await this.bumpAttempts(row);
+ }
+ }
+
+ /** Count the failed replay on the row. Best-effort: the row is already on record either way. */
+ private async bumpAttempts(row: WebhookDeliveryFailure): Promise<void> {
+ try {
+ await this.failureRepository.update({ id: row.id }, { attempts: row.attempts + 1 });
+ } catch (error) {
+ this.logger.warn('Could not record a failed webhook redrive on its row', {
+ failureId: row.id,
+ error: error instanceof Error ? error.message : String(error),
+ action: 'webhook_redrive_bookkeeping_failed',
+ });
+ }
+ }
+}
diff --git a/src/modules/webhook/webhook.controller.spec.ts b/src/modules/webhook/webhook.controller.spec.ts
index f93991348..5952b55ce 100644
--- a/src/modules/webhook/webhook.controller.spec.ts
+++ b/src/modules/webhook/webhook.controller.spec.ts
@@ -6,7 +6,10 @@ import { WebhookService } from './webhook.service';
import { Webhook } from './entities/webhook.entity';
import { Session } from '../session/entities/session.entity';
import { REQUIRED_ROLE_KEY } from '../auth/decorators/auth.decorators';
-import { ApiKeyRole } from '../auth/entities/api-key.entity';
+import { ApiKey, ApiKeyRole } from '../auth/entities/api-key.entity';
+import { WebhookRedriveService } from './webhook-redrive.service';
+import { AuditService } from '../audit/audit.service';
+import { AuditAction } from '../audit/entities/audit-log.entity';
/**
* Regression locks for the secret/headers leak and read authorization at the controller level.
@@ -36,6 +39,8 @@ describe('Webhook controllers (secret leak + read authz)', () => {
let listController: WebhooksListController;
let reflector: Reflector;
let service: jest.Mocked<Partial<WebhookService>>;
+ let redrive: { redrive: jest.Mock };
+ let audit: { logInfo: jest.Mock };
beforeEach(async () => {
service = {
@@ -46,9 +51,16 @@ describe('Webhook controllers (secret leak + read authz)', () => {
update: jest.fn(),
};
+ redrive = { redrive: jest.fn() };
+ audit = { logInfo: jest.fn().mockResolvedValue(null) };
+
const module: TestingModule = await Test.createTestingModule({
controllers: [WebhookController, WebhooksListController],
- providers: [{ provide: WebhookService, useValue: service }],
+ providers: [
+ { provide: WebhookService, useValue: service },
+ { provide: WebhookRedriveService, useValue: redrive },
+ { provide: AuditService, useValue: audit },
+ ],
}).compile();
controller = module.get<WebhookController>(WebhookController);
@@ -137,4 +149,29 @@ describe('Webhook controllers (secret leak + read authz)', () => {
const role = reflector.get<ApiKeyRole>(REQUIRED_ROLE_KEY, listController.findAll);
expect(role).toBe(ApiKeyRole.OPERATOR);
});
+
+ // ── redrive sends real events: ADMIN only, scoped by the key, audited ─────
+
+ it('delivery-failure redrive requires ADMIN role', () => {
+ // eslint-disable-next-line @typescript-eslint/unbound-method -- reading route metadata, not invoking
+ const role = reflector.get<ApiKeyRole>(REQUIRED_ROLE_KEY, listController.redriveDeliveryFailures);
+ expect(role).toBe(ApiKeyRole.ADMIN);
+ });
+
+ it("delivery-failure redrive passes the key's allowedSessions and audits the counts", async () => {
+ const result = { redriven: 2, delivered: 2, enqueued: 0, failed: 1, skipped: 0, remaining: 4 };
+ redrive.redrive.mockResolvedValue(result);
+ const apiKey = { id: 'key-1', allowedSessions: ['sess-1'] } as unknown as ApiKey;
+
+ await expect(listController.redriveDeliveryFailures({ sessionId: 'sess-1', limit: 10 }, apiKey)).resolves.toBe(
+ result,
+ );
+
+ expect(redrive.redrive).toHaveBeenCalledWith({ sessionId: 'sess-1', limit: 10 }, ['sess-1']);
+ expect(audit.logInfo).toHaveBeenCalledWith(AuditAction.WEBHOOK_DELIVERIES_REDRIVEN, {
+ apiKey,
+ sessionId: 'sess-1',
+ metadata: { webhookId: undefined, ids: undefined, redriven: 2, failed: 1, skipped: 0, remaining: 4 },
+ });
+ });
});
diff --git a/src/modules/webhook/webhook.module.ts b/src/modules/webhook/webhook.module.ts
index 0e86cdf47..6c538f6e8 100644
--- a/src/modules/webhook/webhook.module.ts
+++ b/src/modules/webhook/webhook.module.ts
@@ -8,6 +8,7 @@ import { WebhookReconcilerService } from './webhook-reconciler.service';
import { Session } from '../session/entities/session.entity';
import { WebhookService } from './webhook.service';
import { WebhookDeliveryService } from './webhook-delivery.service';
+import { WebhookRedriveService } from './webhook-redrive.service';
import { WebhookController } from './webhook.controller';
import { WebhooksListController } from './webhooks-list.controller';
import { EngineModule } from '../../engine/engine.module';
@@ -29,7 +30,13 @@ if (process.env.QUEUE_ENABLED === 'true') {
...queueModules,
],
controllers: [WebhookController, WebhooksListController],
- providers: [WebhookService, WebhookDeliveryService, WebhookOutboxService, WebhookReconcilerService],
+ providers: [
+ WebhookService,
+ WebhookDeliveryService,
+ WebhookOutboxService,
+ WebhookReconcilerService,
+ WebhookRedriveService,
+ ],
exports: [WebhookService],
})
export class WebhookModule {}
diff --git a/src/modules/webhook/webhook.service.spec.ts b/src/modules/webhook/webhook.service.spec.ts
index 0a692d4f2..6814ac64d 100644
--- a/src/modules/webhook/webhook.service.spec.ts
+++ b/src/modules/webhook/webhook.service.spec.ts
@@ -14,7 +14,7 @@ import { createHmac } from 'crypto';
import { Test, TestingModule } from '@nestjs/testing';
import { getRepositoryToken } from '@nestjs/typeorm';
import { getQueueToken } from '@nestjs/bullmq';
-import { In, Repository } from 'typeorm';
+import { FindOperator, In, Repository } from 'typeorm';
import { NotFoundException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { fetch as undiciFetch } from 'undici';
@@ -75,6 +75,7 @@ describe('WebhookService', () => {
insert: jest.fn().mockResolvedValue({}),
find: jest.fn().mockResolvedValue([]),
delete: jest.fn().mockResolvedValue({ affected: 0 }),
+ update: jest.fn().mockResolvedValue({ affected: 0 }),
};
sessionRepository = {
@@ -504,6 +505,54 @@ describe('WebhookService', () => {
else process.env.WEBHOOK_FAILURE_RETENTION_DAYS = prev;
}
});
+
+ it('pruneDeliveryFailurePayloads clears expired payloads and keeps the rows', async () => {
+ (failureRepository.update as jest.Mock).mockResolvedValue({ affected: 2 });
+ const before = Date.now();
+
+ await expect(service.pruneDeliveryFailurePayloads(72)).resolves.toBe(2);
+
+ const [where, patch] = (failureRepository.update as jest.Mock).mock.calls[0] as [
+ { payload: FindOperator<unknown>; createdAt: FindOperator<Date> },
+ Record<string, unknown>,
+ ];
+ expect(patch).toEqual({ payload: null });
+ expect(where.payload.type).toBe('not');
+ expect(where.createdAt.type).toBe('lessThan');
+ const cutoff = where.createdAt.value.getTime();
+ expect(cutoff).toBeLessThanOrEqual(before - 72 * 3600_000 + 1000);
+ expect(cutoff).toBeGreaterThanOrEqual(before - 72 * 3600_000 - 1000);
+ expect(failureRepository.delete).not.toHaveBeenCalled();
+ });
+
+ it('with payload retention off, startup clears leftover payloads once and schedules nothing', () => {
+ jest.useFakeTimers();
+ try {
+ service.onModuleInit();
+ expect(failureRepository.update).toHaveBeenCalledTimes(1);
+ jest.advanceTimersByTime(2 * 60 * 60 * 1000);
+ expect(failureRepository.update).toHaveBeenCalledTimes(1);
+ } finally {
+ service.onModuleDestroy();
+ jest.useRealTimers();
+ }
+ });
+
+ it('with payload retention on, expired payloads are cleared at startup and then hourly', () => {
+ (configService.get as jest.Mock).mockImplementation(<T>(key: string, def?: T) =>
+ key === 'webhook.failurePayloadRetentionHours' ? 24 : def,
+ );
+ jest.useFakeTimers();
+ try {
+ service.onModuleInit();
+ expect(failureRepository.update).toHaveBeenCalledTimes(1);
+ jest.advanceTimersByTime(60 * 60 * 1000);
+ expect(failureRepository.update).toHaveBeenCalledTimes(2);
+ } finally {
+ service.onModuleDestroy();
+ jest.useRealTimers();
+ }
+ });
});
// ── dispatch facade ───────────────────────────────────────────────
@@ -628,5 +677,30 @@ describe('WebhookService', () => {
expect.objectContaining({ where: { sessionId: In(['s1']) }, order: { createdAt: 'DESC', id: 'DESC' } }),
);
});
+
+ it('flags the rows that still hold a replay payload, without reading any payload', async () => {
+ (failureRepository.find as jest.Mock)
+ .mockResolvedValueOnce([{ id: 'f1' }, { id: 'f2' }])
+ .mockResolvedValueOnce([{ id: 'f2' }]);
+
+ const out = await service.listDeliveryFailures({});
+
+ expect(out).toEqual([
+ { id: 'f1', replayable: false },
+ { id: 'f2', replayable: true },
+ ]);
+ const second = ((failureRepository.find as jest.Mock).mock.calls as unknown[][])[1][0] as {
+ select: Record<string, boolean>;
+ where: Record<string, unknown>;
+ };
+ expect(second.select).toEqual({ id: true });
+ expect(second.where.id).toEqual(In(['f1', 'f2']));
+ });
+
+ it('skips the replayable lookup for an empty page', async () => {
+ (failureRepository.find as jest.Mock).mockResolvedValue([]);
+ await expect(service.listDeliveryFailures({})).resolves.toEqual([]);
+ expect(failureRepository.find).toHaveBeenCalledTimes(1);
+ });
});
});
diff --git a/src/modules/webhook/webhook.service.ts b/src/modules/webhook/webhook.service.ts
index c0c582580..12a9e6971 100644
--- a/src/modules/webhook/webhook.service.ts
+++ b/src/modules/webhook/webhook.service.ts
@@ -1,6 +1,6 @@
import { Injectable, NotFoundException, BadRequestException, OnModuleInit, OnModuleDestroy } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
-import { FindManyOptions, In, LessThan, Repository } from 'typeorm';
+import { FindManyOptions, In, IsNull, LessThan, MoreThan, Not, Repository } from 'typeorm';
import { ConfigService } from '@nestjs/config';
import { Webhook } from './entities/webhook.entity';
import { WebhookDeliveryFailure } from './entities/webhook-delivery-failure.entity';
@@ -42,6 +42,7 @@ const DEFAULT_WEBHOOK_MAX_PER_SESSION = 16;
export class WebhookService implements OnModuleInit, OnModuleDestroy {
private readonly logger = createLogger('WebhookService');
private cleanupTimer?: ReturnType<typeof setInterval>;
+ private payloadCleanupTimer?: ReturnType<typeof setInterval>;
constructor(
@InjectRepository(Webhook, 'data')
@@ -61,6 +62,7 @@ export class WebhookService implements OnModuleInit, OnModuleDestroy {
* without bound under a receiver outage. (Mirrors AuditService's audit-log retention.)
*/
onModuleInit(): void {
+ this.schedulePayloadPrune();
const parsed = Number.parseInt(process.env.WEBHOOK_FAILURE_RETENTION_DAYS ?? '', 10);
const retentionDays = Number.isInteger(parsed) ? Math.max(0, parsed) : 90;
if (retentionDays <= 0) {
@@ -85,6 +87,9 @@ export class WebhookService implements OnModuleInit, OnModuleDestroy {
if (this.cleanupTimer) {
clearInterval(this.cleanupTimer);
}
+ if (this.payloadCleanupTimer) {
+ clearInterval(this.payloadCleanupTimer);
+ }
}
/**
@@ -97,6 +102,47 @@ export class WebhookService implements OnModuleInit, OnModuleDestroy {
return result.affected || 0;
}
+ /**
+ * Clear the replay payload of every failure row recorded more than `olderThanHours` ago; the row
+ * itself stays for WEBHOOK_FAILURE_RETENTION_DAYS. 0 clears every stored payload. Returns the
+ * number of rows cleared.
+ */
+ async pruneDeliveryFailurePayloads(olderThanHours: number): Promise<number> {
+ const cutoff = new Date(Date.now() - olderThanHours * 60 * 60 * 1000);
+ const result = await this.failureRepository.update(
+ { payload: Not(IsNull()), createdAt: LessThan(cutoff) },
+ { payload: null },
+ );
+ return result.affected || 0;
+ }
+
+ /**
+ * WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS bounds how long a failure row holds a whole event body.
+ * With the knob on, expired payloads are cleared at startup and then hourly. With it off (0, the
+ * default) nothing new is stored, and one startup pass clears what an earlier setting left behind,
+ * so turning the feature off also stops keeping the bodies it already stored.
+ */
+ private schedulePayloadPrune(): void {
+ const hours = this.configService.get<number>('webhook.failurePayloadRetentionHours', 0);
+ const runPrune = (): void => {
+ this.pruneDeliveryFailurePayloads(hours)
+ .then(n => {
+ if (n > 0) this.logger.log(`Cleared the replay payload of ${n} webhook delivery-failure(s)`);
+ })
+ .catch(err =>
+ this.logger.error(
+ 'Webhook delivery-failure payload cleanup failed',
+ err instanceof Error ? err.stack : String(err),
+ ),
+ );
+ };
+ runPrune();
+ if (hours > 0) {
+ this.payloadCleanupTimer = setInterval(runPrune, 60 * 60 * 1000);
+ this.payloadCleanupTimer.unref?.();
+ }
+ }
+
/**
* Reject an internal/unsafe webhook URL at registration, so a bad URL fails
* synchronously with a 400 instead of silently failing at delivery time. Honors the same
@@ -195,17 +241,33 @@ export class WebhookService implements OnModuleInit, OnModuleDestroy {
async listDeliveryFailures(
opts: ListOptions & { sessionId?: string } = {},
allowedSessions?: string[] | null,
- ): Promise<WebhookDeliveryFailure[]> {
+ ): Promise<Array<WebhookDeliveryFailure & { replayable: boolean }>> {
const { limit, offset } = resolveListWindow(opts.limit, opts.offset);
const sessionScope = resolveSessionScope(allowedSessions, opts.sessionId);
if (sessionScope !== null && sessionScope.length === 0) return []; // requested session outside the key's scope
- return this.failureRepository.find({
+ const rows = await this.failureRepository.find({
where: sessionScope ? { sessionId: In(sessionScope) } : {},
// A receiver outage writes a burst of failures inside one second; `id` keeps the page order total.
order: { createdAt: 'DESC', id: 'DESC' },
take: limit,
skip: offset,
});
+ // `payload` is select: false, so the page above never carries an event body. Which rows hold one
+ // is a second, id-only read; with payload retention off no row does and it matches nothing.
+ const replayable = new Set<string>();
+ if (rows.length > 0) {
+ const withPayload = await this.failureRepository.find({
+ select: { id: true },
+ where: {
+ id: In(rows.map(r => r.id)),
+ payload: Not(IsNull()),
+ attempts: MoreThan(0),
+ idempotencyKey: Not(IsNull()),
+ },
+ });
+ for (const r of withPayload) replayable.add(r.id);
+ }
+ return rows.map(r => Object.assign(r, { replayable: replayable.has(r.id) }));
}
async findOne(sessionId: string, id: string): Promise<Webhook> {
diff --git a/src/modules/webhook/webhooks-list.controller.ts b/src/modules/webhook/webhooks-list.controller.ts
index b3b442652..999726474 100644
--- a/src/modules/webhook/webhooks-list.controller.ts
+++ b/src/modules/webhook/webhooks-list.controller.ts
@@ -1,15 +1,27 @@
-import { Controller, Get, Query } from '@nestjs/common';
+import { Body, Controller, Get, HttpCode, HttpStatus, Post, Query } from '@nestjs/common';
import { ApiTags, ApiOperation, ApiResponse, ApiQuery } from '@nestjs/swagger';
import { WebhookService } from './webhook.service';
-import { WebhookResponseDto, WebhookDeliveryFailureDto } from './dto';
+import { WebhookRedriveResult, WebhookRedriveService } from './webhook-redrive.service';
+import {
+ WebhookResponseDto,
+ WebhookDeliveryFailureDto,
+ RedriveWebhookDeliveriesDto,
+ WebhookRedriveResultDto,
+} from './dto';
import { WebhookDeliveryFailure } from './entities/webhook-delivery-failure.entity';
import { RequireRole, CurrentApiKey } from '../auth/decorators/auth.decorators';
import { ApiKey, ApiKeyRole } from '../auth/entities/api-key.entity';
+import { AuditService } from '../audit/audit.service';
+import { AuditAction } from '../audit/entities/audit-log.entity';
@ApiTags('webhooks')
@Controller('webhooks')
export class WebhooksListController {
- constructor(private readonly webhookService: WebhookService) {}
+ constructor(
+ private readonly webhookService: WebhookService,
+ private readonly redriveService: WebhookRedriveService,
+ private readonly audit: AuditService,
+ ) {}
@Get('delivery-failures')
@RequireRole(ApiKeyRole.ADMIN)
@@ -32,7 +44,7 @@ export class WebhooksListController {
@Query('sessionId') sessionId?: string,
@Query('limit') limit?: string,
@Query('offset') offset?: string,
- ): Promise<WebhookDeliveryFailure[]> {
+ ): Promise<Array<WebhookDeliveryFailure & { replayable: boolean }>> {
// Scope to the calling key's allowedSessions so a session-restricted ADMIN key cannot read another
// session's failed-delivery URLs/errors via the `sessionId` query param (which bypasses the guard).
return this.webhookService.listDeliveryFailures(
@@ -45,6 +57,45 @@ export class WebhooksListController {
);
}
+ // Replaying lost deliveries sends real events to receivers, so it is ADMIN-gated like the listing
+ // and the integration redrive. The body's sessionId bypasses the guard's route-param fence, so the
+ // service scopes every row to the calling key's allowedSessions itself.
+ @Post('delivery-failures/redrive')
+ @RequireRole(ApiKeyRole.ADMIN)
+ @HttpCode(HttpStatus.OK)
+ @ApiOperation({ summary: 'Replay recorded webhook deliveries that still hold their event data' })
+ @ApiResponse({
+ status: 200,
+ description:
+ 'One bounded batch of replayable failure rows (terminal rows recorded while ' +
+ 'WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS > 0), oldest first. Each replay reuses the stored ' +
+ 'idempotency key, so a receiver that already handled the event can dedup it, and runs the ' +
+ 'webhook:before hooks again. With the queue disabled each row gets one attempt; with it enabled ' +
+ 'the row is enqueued with the webhook retry policy. Rows outside the key allowedSessions are ' +
+ 'never touched.',
+ type: WebhookRedriveResultDto,
+ })
+ async redriveDeliveryFailures(
+ @Body() dto: RedriveWebhookDeliveriesDto,
+ @CurrentApiKey() apiKey?: ApiKey,
+ ): Promise<WebhookRedriveResult> {
+ const result = await this.redriveService.redrive(dto, apiKey?.allowedSessions);
+ // A redrive sends real events, so every batch is audited with its counts (no payload content).
+ void this.audit.logInfo(AuditAction.WEBHOOK_DELIVERIES_REDRIVEN, {
+ apiKey,
+ sessionId: dto.sessionId,
+ metadata: {
+ webhookId: dto.webhookId,
+ ids: dto.ids?.length,
+ redriven: result.redriven,
+ failed: result.failed,
+ skipped: result.skipped,
+ remaining: result.remaining,
+ },
+ });
+ return result;
+ }
+
@Get()
@RequireRole(ApiKeyRole.OPERATOR)
@ApiOperation({ summary: 'List webhooks visible to the calling key (scoped to its allowed sessions)' })