Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(webhook): keep a failing receiver from holding every delivery slot
Yudhi Armyndharisfeb2da5cc611
diff --git a/.env.example b/.env.exampleindex 3709185fb..75684a4c0 100644--- a/.env.example+++ b/.env.example@@ -424,8 +424,13 @@ DATABASE_SYNCHRONIZE=false # WARNING: Set false in production! (data DB# =============================================================================WEBHOOK_TIMEOUT=10000 # Timeout in millisecondsWEBHOOK_RETRY_DELAY=5000 # Delay between retries in ms-# WEBHOOK_DISPATCH_CONCURRENCY=16 # max active inline webhook deliveries per process-# WEBHOOK_DISPATCH_MAX_QUEUED=1000 # max inline deliveries parked behind the concurrency cap+# WEBHOOK_DISPATCH_CONCURRENCY=16 # max inline webhook POSTs in flight per process; a delivery+ # waiting out a retry backoff does not hold one+# WEBHOOK_DISPATCH_MAX_QUEUED=1000 # inline deliveries running, parked, or waiting out a retry+ # backoff are capped together at WEBHOOK_DISPATCH_CONCURRENCY+ # plus this; more are shed to the outbox.+ # Deliveries to one session's failing receivers park at most+ # a quarter of it, so other sessions keep room to wait# WEBHOOK_SHUTDOWN_DRAIN_MS=5000 # on shutdown, in-flight deliveries get this long to finish# before they are logged abandoned (parked ones are# dead-lettered). Startup warns when this is shorter than@@ -737,13 +742,22 @@ WEBHOOK_SSRF_PROTECT=true# when QUEUE_ENABLED=true. Ordering within one conversation is preserved by a per-conversation lock,# so raising these parallelizes unrelated conversations rather than reordering them. Neither pool is# isolated per webhook or per conversation:-# - A dead webhook receiver holds a slot for up to WEBHOOK_TIMEOUT per attempt, retries included, so-# keep WEBHOOK_WORKER_CONCURRENCY above its events per second x attempts x WEBHOOK_TIMEOUT (in-# seconds), or every other webhook waits behind it.+# - A dead webhook receiver holds a slot for up to WEBHOOK_TIMEOUT per attempt. Once a webhook's+# attempt fails, jobs for it that start afterwards take a per-session slot, at most+# WEBHOOK_DEGRADED_SESSION_CONCURRENCY at a time (the rest wait in the delayed set without+# spending an attempt); a job already running when the failure lands is not counted. The first+# 2xx lifts the limit. Keep WEBHOOK_WORKER_CONCURRENCY above WEBHOOK_DEGRADED_SESSION_CONCURRENCY+# times the number of sessions whose receivers may fail at once, or those sessions fill the+# pool and every other webhook waits behind them.# - An ingress job waiting on a busy conversation still holds a slot, so keep# INGRESS_WORKER_CONCURRENCY above the largest burst on one conversation, and declare a# conversationId pointer on the route (without one, the whole instance is one conversation).# WEBHOOK_WORKER_CONCURRENCY=10 # webhook delivery workers (default 10)+# WEBHOOK_DEGRADED_SESSION_CONCURRENCY= # per-session cap on deliveries to failing webhook receivers,+ # queued, and inline with QUEUE_ENABLED=false (the direct+ # fallback for a rejected enqueue is not capped);+ # default a quarter of the pool+ # (WEBHOOK_WORKER_CONCURRENCY or WEBHOOK_DISPATCH_CONCURRENCY)# INGRESS_WORKER_CONCURRENCY=10 # integration ingress workers (default 10)# Opt-in to load plugins whose ingress routes declare signature.scheme 'none'.diff --git a/docker-compose.dev.yml b/docker-compose.dev.ymlindex 9649b42d2..9e3272b6e 100644--- a/docker-compose.dev.yml+++ b/docker-compose.dev.yml@@ -207,6 +207,7 @@ services:- WEBHOOK_MAX_PER_SESSION=${WEBHOOK_MAX_PER_SESSION:-}- WEBHOOK_FAILURE_RETENTION_DAYS=${WEBHOOK_FAILURE_RETENTION_DAYS:-}- WEBHOOK_WORKER_CONCURRENCY=${WEBHOOK_WORKER_CONCURRENCY:-}+ - WEBHOOK_DEGRADED_SESSION_CONCURRENCY=${WEBHOOK_DEGRADED_SESSION_CONCURRENCY:-}# Integration Fabric ingress: fairness, retries, retention, reconciler, workers.- INGRESS_INSTANCE_LIMIT=${INGRESS_INSTANCE_LIMIT:-}- INGRESS_INSTANCE_TTL=${INGRESS_INSTANCE_TTL:-}diff --git a/docker-compose.yml b/docker-compose.ymlindex adfb4fc15..71cb28fe9 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -304,6 +304,7 @@ services:- WEBHOOK_MAX_PER_SESSION=${WEBHOOK_MAX_PER_SESSION:-}- WEBHOOK_FAILURE_RETENTION_DAYS=${WEBHOOK_FAILURE_RETENTION_DAYS:-}- WEBHOOK_WORKER_CONCURRENCY=${WEBHOOK_WORKER_CONCURRENCY:-}+ - WEBHOOK_DEGRADED_SESSION_CONCURRENCY=${WEBHOOK_DEGRADED_SESSION_CONCURRENCY:-}# Integration Fabric ingress: per-instance fairness, retries, retention, reconciler, workers.- INGRESS_INSTANCE_LIMIT=${INGRESS_INSTANCE_LIMIT:-}- INGRESS_INSTANCE_TTL=${INGRESS_INSTANCE_TTL:-}diff --git a/docs/06-api-specification.md b/docs/06-api-specification.mdindex f795b1fff..9231d7257 100644--- a/docs/06-api-specification.md+++ b/docs/06-api-specification.md@@ -4624,7 +4624,7 @@ List webhook deliveries that exhausted every retry, most recent first. This is tBare array of `WebhookDeliveryFailure` rows, ordered by `createdAt` descending. `lastStatusCode` is `null` when the failure was a network/timeout/SSRF error rather than a non-2xx response; `idempotencyKey`/`deliveryId` let you correlate the lost event with your own receiver logs.-A delivery shed because the dispatch queue was full, or refused during shutdown, is recorded here with `attempts: 0` while its outbox row stays pending, so the outbox sweep replays it. Any delivery the receiver accepts with a `2xx`, inline or from a queued job, removes the rows filed under its idempotency key, so a replay that succeeds leaves none. A replay that exhausts its retries files a row carrying the real error and attempt count and only then removes the `attempts: 0` row, so the event stays on record throughout, across a restart or a lost queue job too. A replay that fails before sending (a payload over the size cap after the `webhook:before` hooks, or one that cannot be serialized) gives the `attempts: 0` row that reason instead. A replay that keeps failing files no second row. `openwa_webhook_delivery_failures_total` counts the original shed, and counts once more if a replay also exhausts its retries.+A delivery shed because the dispatch queue was full, or refused during shutdown, is recorded here with `attempts: 0` while its outbox row stays pending, so the outbox sweep replays it. With the queue disabled, a delivery that shutdown interrupts while it waits out a retry backoff is recorded the same way (`attempts: 0`, `lastError` `ConcurrencyLimiter closed`) if the backoff ends within `WEBHOOK_SHUTDOWN_DRAIN_MS`, although its earlier attempts were sent and their responses are not in the row; one still asleep when the drain ends is only logged (`webhook_delivery_abandoned_shutdown`). Either way its outbox row stays pending, and the replay on the next start replaces or clears the row. Any delivery the receiver accepts with a `2xx`, inline or from a queued job, removes the rows filed under its idempotency key, so a replay that succeeds leaves none. A replay that exhausts its retries files a row carrying the real error and attempt count and only then removes the `attempts: 0` row, so the event stays on record throughout, across a restart or a lost queue job too. A replay that fails before sending (a payload over the size cap after the `webhook:before` hooks, or one that cannot be serialized) gives the `attempts: 0` row that reason instead. A replay that keeps failing files no second row. `openwa_webhook_delivery_failures_total` counts the original shed, and counts once more if a replay also exhausts its retries.**Errors:** `401` missing/invalid API key · `403` key role below ADMIN@@ -7076,7 +7076,7 @@ Webhook delivery is **at-least-once**. A consumer can legitimately receive the s- The underlying WhatsApp engine can re-fire an event for a single message.- A failed delivery (non-2xx response, timeout, or network error) is retried.-**Crash boundary.** Every delivery is recorded before it is attempted, and the record is retired once something durable owns it: the queue job in queued mode, the completed send in direct mode. A hard crash (SIGKILL, OOM) therefore leaves the record behind, and a bounded sweep (`WEBHOOK_RECONCILE_INTERVAL_MS`, default 60s) replays whatever is still stranded, reusing the stored `X-OpenWA-Idempotency-Key` so the retry stays deduplicable at your receiver. A delivery that keeps failing exhausts `WEBHOOK_RECONCILE_MAX_ATTEMPTS` and goes terminal rather than replaying forever. One window remains open: a crash between persisting the message and writing that record loses the delivery, because the two are not yet one transaction. The failure table records exhausted retries, plus over-budget, dispatch-capacity-exceeded, and shutdown-rejected deliveries with attempts 0. Read the last two as a report rather than a verdict: a delivery the dispatcher shed for capacity or refused during the drain was rejected before its POST, so it keeps its record until the same sweep replays it; a replay whose retries run out replaces that record with its own row, one that fails before sending gives it that reason, and one that delivers removes it. A node's sweep also leaves alone a delivery that the same node is still waiting to dispatch or retrying, however long that takes; with several nodes on one database, another node can still replay it, and the stored idempotency key keeps that duplicate deduplicable. Enabling the queue (`QUEUE_ENABLED=true`, needs Redis) makes the dispatch durable from the enqueue onward. In both modes, a graceful shutdown drains in-flight deliveries first: the queued path waits for each worker's current job, and the direct path waits up to `WEBHOOK_SHUTDOWN_DRAIN_MS` (default 5s; raise it to at least `WEBHOOK_TIMEOUT`, default 10s, if a slow receiver must finish).+**Crash boundary.** Every delivery is recorded before it is attempted, and the record is retired once something durable owns it: the queue job in queued mode, the completed send in direct mode. A hard crash (SIGKILL, OOM) therefore leaves the record behind, and a bounded sweep (`WEBHOOK_RECONCILE_INTERVAL_MS`, default 60s) replays whatever is still stranded, reusing the stored `X-OpenWA-Idempotency-Key` so the retry stays deduplicable at your receiver. A delivery that keeps failing exhausts `WEBHOOK_RECONCILE_MAX_ATTEMPTS` and goes terminal rather than replaying forever. One window remains open: a crash between persisting the message and writing that record loses the delivery, because the two are not yet one transaction. The failure table records exhausted retries, plus over-budget, dispatch-capacity-exceeded, and shutdown-rejected deliveries with attempts 0. Read the last two as a report rather than a verdict: a delivery the dispatcher shed for capacity or refused during the drain was rejected before its POST, except a direct delivery the drain caught in a retry backoff, whose earlier attempts were sent, so it keeps its record until the same sweep replays it; a replay whose retries run out replaces that record with its own row, one that fails before sending gives it that reason, and one that delivers removes it. A node's sweep also leaves alone a delivery that the same node is still waiting to dispatch or retrying, however long that takes; with several nodes on one database, another node can still replay it, and the stored idempotency key keeps that duplicate deduplicable. Enabling the queue (`QUEUE_ENABLED=true`, needs Redis) makes the dispatch durable from the enqueue onward. In both modes, a graceful shutdown drains in-flight deliveries first: the queued path waits for each worker's current job, and the direct path waits up to `WEBHOOK_SHUTDOWN_DRAIN_MS` (default 5s; raise it to at least `WEBHOOK_TIMEOUT`, default 10s, if a slow receiver must finish).**Design your handler to be idempotent**, keyed on the `X-OpenWA-Idempotency-Key` header (see below). As a server-side safety net, OpenWA de-duplicates inbound `message.received` before dispatch (a re-fired event for an already-persisted message is dropped), so one webhook normally sees each inbound message once — but this is best-effort defense-in-depth and does not remove the need for consumer-side idempotency.@@ -7141,7 +7141,7 @@ Recurring lifecycle events (and `message.reaction` / `message.edited`) carry the### Retries with exponential backoff-When the queue is enabled, a non-2xx response, timeout (`WEBHOOK_TIMEOUT`, default `10000` ms), or network error schedules a retry. The number of attempts comes from the webhook's `retryCount` (default `3`) and the delay grows **exponentially** from a base of `WEBHOOK_RETRY_DELAY` (default `5000` ms). Each retry reuses the same `idempotencyKey` and increments `X-OpenWA-Retry-Count`. If Redis/BullMQ rejects the initial enqueue, OpenWA logs a `webhook:error` hook event and falls back to direct delivery with the same inline retry budget. When the queue is disabled, delivery is direct with the same retry budget and the same exponential backoff applied inline. Every retry, queued or direct, re-reads the webhook first: a webhook deleted, disabled or unsubscribed from the event since the dispatch gets no further attempt and no delivery-failure row, and a changed URL, secret or header map applies from the next attempt.+When the queue is enabled, a non-2xx response, timeout (`WEBHOOK_TIMEOUT`, default `10000` ms), or network error schedules a retry. The number of attempts comes from the webhook's `retryCount` (default `3`) and the delay grows **exponentially** from a base of `WEBHOOK_RETRY_DELAY` (default `5000` ms). Each retry reuses the same `idempotencyKey` and increments `X-OpenWA-Retry-Count`. If Redis/BullMQ rejects the initial enqueue, OpenWA logs a `webhook:error` hook event and falls back to direct delivery with the same inline retry budget. When the queue is disabled, delivery is direct with the same retry budget and the same exponential backoff applied inline. Every retry, queued or direct, re-reads the webhook first: a webhook deleted, disabled or unsubscribed from the event since the dispatch gets no further attempt and no delivery-failure row, and a changed URL, secret or header map applies from the next attempt. With the queue disabled, a delivery that waited behind the failing-webhook cap (`WEBHOOK_DEGRADED_SESSION_CONCURRENCY`) re-reads the webhook the same way before its first attempt; if that read fails, the attempt uses the webhook as loaded at dispatch.### SSRF guard on registrationdiff --git a/docs/10-devops-infrastructure.md b/docs/10-devops-infrastructure.mdindex b0e370da6..da9dbb00a 100644--- a/docs/10-devops-infrastructure.md+++ b/docs/10-devops-infrastructure.md@@ -495,6 +495,22 @@ WEBHOOK_TIMEOUT=10000WEBHOOK_RETRY_DELAY=5000WEBHOOK_DISPATCH_CONCURRENCY=16WEBHOOK_DISPATCH_MAX_QUEUED=1000+# WEBHOOK_DISPATCH_CONCURRENCY caps inline POSTs in flight; a delivery waiting out a retry backoff+# holds no slot. Running, parked, and backoff-waiting inline deliveries together are capped at+# WEBHOOK_DISPATCH_CONCURRENCY + WEBHOOK_DISPATCH_MAX_QUEUED; past that bound a new delivery is+# shed and left for the outbox sweep. Once a webhook's attempt fails, this process caps that+# session's deliveries to it at WEBHOOK_DEGRADED_SESSION_CONCURRENCY at once+# (default: a quarter of WEBHOOK_DISPATCH_CONCURRENCY or WEBHOOK_WORKER_CONCURRENCY, at least 1).+# With the queue disabled, the cap applies to inline deliveries admitted after the failure; ones+# admitted before it (running, parked, or in their retries) are not counted. The direct delivery+# used when Redis rejects an enqueue is not capped. Queued, it applies to every job attempt+# that starts afterwards, including retries and jobs already waiting (the rest wait in the delayed+# set without spending an attempt); only an attempt already running is not counted. The failing+# state and the cap are held per process, not per cluster.+# The first 2xx from the webhook lifts it. With the queue disabled, a session parks at most a+# quarter of WEBHOOK_DISPATCH_MAX_QUEUED behind that limit and sheds the rest, so other sessions+# keep room.+# WEBHOOK_DEGRADED_SESSION_CONCURRENCY=# Delivery attempts (total, including the first) are set per webhook with the retryCount API field (default 3, range 0-5).# ===========================================@@ -940,29 +956,29 @@ export class MetricsService {**Exported metric names** (the complete set — nothing else is emitted):-| Metric | Type | Labels | Meaning |-| -------------------------------------------- | --------- | ----------------------------------- | -------------------------------------------------------------------------------------------- |-| `openwa_up` | gauge | — | Always `1` when scraped |-| `openwa_process_uptime_seconds` | gauge | — | Process uptime |-| `openwa_process_resident_memory_bytes` | gauge | — | RSS |-| `openwa_process_heap_used_bytes` | gauge | — | V8 heap used |-| `openwa_event_loop_delay_p99_seconds` | gauge | none | p99 event-loop delay since the previous uncached scrape |-| `openwa_event_loop_delay_max_seconds` | gauge | none | Maximum event-loop delay since the previous uncached scrape |-| `openwa_unhandled_rejections_total` | counter | `kind` | Unhandled promise rejections since process start (`other` or `page_context_lost`) |-| `openwa_queue_jobs` | gauge | `queue`, `state` | BullMQ jobs per queue in `wait`/`active`/`delayed`/`failed` (cluster-wide, from Redis) |-| `openwa_stats_available` | gauge | — | 1 when the last overview read of the database-derived series below succeeded, 0 if it failed |-| `openwa_sessions_total` | gauge | — | Configured sessions |-| `openwa_sessions_active` | gauge | — | READY (active) sessions |-| `openwa_sessions` | gauge | `status` | Session count per status |-| `openwa_messages_total` | gauge | `direction` (`incoming`/`outgoing`) | Current stored messages by direction |-| `openwa_messages_failed_total` | gauge | — | Current messages in FAILED state |-| `openwa_webhook_delivery_failures_total` | counter | — | Webhook delivery failures since process start: retries exhausted, or never sent |-| `openwa_session_reconnect_attempts_total` | counter | — | Reconnect attempts scheduled across all sessions since process start |-| `openwa_session_reconnect_loop_alerts_total` | counter | — | Reconnect-loop alerts emitted since process start |-| `openwa_sessions_restricted` | gauge | — | Sessions whose account WhatsApp is currently restricting |-| `openwa_send_pacing_refusals_total` | counter | `reason` | Sends refused by the pacing governor since process start |-| `http_requests_total` | counter | `method`, `route`, `status` | HTTP requests served, by method, route and status |-| `http_request_duration_seconds` | histogram | `method`, `route` | HTTP request duration (`_bucket` / `_sum` / `_count`) |+| Metric | Type | Labels | Meaning |+| -------------------------------------------- | --------- | ----------------------------------- | --------------------------------------------------------------------------------------------------------------------------- |+| `openwa_up` | gauge | — | Always `1` when scraped |+| `openwa_process_uptime_seconds` | gauge | — | Process uptime |+| `openwa_process_resident_memory_bytes` | gauge | — | RSS |+| `openwa_process_heap_used_bytes` | gauge | — | V8 heap used |+| `openwa_event_loop_delay_p99_seconds` | gauge | none | p99 event-loop delay since the previous uncached scrape |+| `openwa_event_loop_delay_max_seconds` | gauge | none | Maximum event-loop delay since the previous uncached scrape |+| `openwa_unhandled_rejections_total` | counter | `kind` | Unhandled promise rejections since process start (`other` or `page_context_lost`) |+| `openwa_queue_jobs` | gauge | `queue`, `state` | BullMQ jobs per queue in `wait`/`active`/`delayed`/`failed` (cluster-wide, from Redis) |+| `openwa_stats_available` | gauge | — | 1 when the last overview read of the database-derived series below succeeded, 0 if it failed |+| `openwa_sessions_total` | gauge | — | Configured sessions |+| `openwa_sessions_active` | gauge | — | READY (active) sessions |+| `openwa_sessions` | gauge | `status` | Session count per status |+| `openwa_messages_total` | gauge | `direction` (`incoming`/`outgoing`) | Current stored messages by direction |+| `openwa_messages_failed_total` | gauge | — | Current messages in FAILED state |+| `openwa_webhook_delivery_failures_total` | counter | — | Webhook delivery failures since process start: retries exhausted, never sent, or stopped by shutdown between direct retries |+| `openwa_session_reconnect_attempts_total` | counter | — | Reconnect attempts scheduled across all sessions since process start |+| `openwa_session_reconnect_loop_alerts_total` | counter | — | Reconnect-loop alerts emitted since process start |+| `openwa_sessions_restricted` | gauge | — | Sessions whose account WhatsApp is currently restricting |+| `openwa_send_pacing_refusals_total` | counter | `reason` | Sends refused by the pacing governor since process start |+| `http_requests_total` | counter | `method`, `route`, `status` | HTTP requests served, by method, route and status |+| `http_request_duration_seconds` | histogram | `method`, `route` | HTTP request duration (`_bucket` / `_sum` / `_count`) |The last two are deliberately **unprefixed** so a generic RED dashboard or alert rule matches themwithout knowing anything about OpenWA. They come from `src/common/metrics/request-metrics.ts`, whichdiff --git a/docs/11-operational-runbooks.md b/docs/11-operational-runbooks.mdindex 1015b0ba5..60b350f89 100644--- a/docs/11-operational-runbooks.md+++ b/docs/11-operational-runbooks.md@@ -252,9 +252,11 @@ curl -H "X-API-Key: $API_KEY" \http://localhost:2785/api/sessions/{sessionId}/webhooks# 2. Check recent webhook deliveries — this admin-only endpoint lists abandoned deliveries-# most-recent-first: those that exhausted every retry, plus those never attempted at all-# (recorded with `attempts: 0` — payload over the cap or an unserializable payload-# (preflight), inline waiter-queue overflow, or rejection by the shutdown drain).+# most-recent-first: those that exhausted every retry, plus those recorded with+# `attempts: 0` — never attempted (payload over the cap or an+# unserializable payload (preflight), inline waiter-queue overflow, or rejection by the+# shutdown drain), or, with the queue disabled, stopped by shutdown in a retry backoff that+# ended within WEBHOOK_SHUTDOWN_DRAIN_MS, after earlier attempts were sent.# An overflow or shutdown row is replayed by the outbox sweep and removed once it delivers.# The SSRF guard refuses a blocked URL with a 400 when the webhook is registered. A URL that# passed then and is blocked at delivery (its host now resolves to a private address, or thediff --git a/docs/12-troubleshooting-faq.md b/docs/12-troubleshooting-faq.mdindex fed295e0a..0569cf5c5 100644--- a/docs/12-troubleshooting-faq.md+++ b/docs/12-troubleshooting-faq.md@@ -898,8 +898,9 @@ curl -X POST http://localhost:2785/api/sessions/{id}/messages/send-image \curl -H "X-API-Key: $API_KEY" \http://localhost:2785/api/sessions/{sessionId}/webhooks-# Abandoned deliveries, most recent first: those that exhausted every retry, plus those never-# attempted at all (recorded with `attempts: 0`). Requires an ADMIN key — an OPERATOR key gets+# Abandoned deliveries, most recent first: those that exhausted every retry, plus those recorded+# with `attempts: 0`: never attempted, or (queue disabled) stopped by shutdown during a retry+# backoff after earlier attempts. Requires an ADMIN key — an OPERATOR key gets# a 403, which reads like the endpoint does not exist. Rows older than# WEBHOOK_FAILURE_RETENTION_DAYS (default 90) are pruned.curl -H "X-API-Key: $ADMIN_API_KEY" \diff --git a/docs/22-n8n-integration.md b/docs/22-n8n-integration.mdindex c3f4cf60c..224ac35b7 100644--- a/docs/22-n8n-integration.md+++ b/docs/22-n8n-integration.md@@ -288,9 +288,11 @@ Always use the correct format for chat IDs:`GET /api/webhooks/delivery-failures?sessionId={sessionId}` (ADMIN key). A row with an HTTP`lastStatusCode` means OpenWA delivered and n8n rejected it. A row without one and with`attempts: 0` was never sent (shed under load, dropped at shutdown, over the payload size cap, or- failed before sending; a shed or shutdown row is replayed later); with attempts, n8n timed out or- was unreachable. An empty list means nothing has failed permanently yet: retries still in flight- show only in the server logs, and an event that never matched the webhook leaves no row+ failed before sending; a shed or shutdown row is replayed later), except one that shutdown+ stopped between retries with the queue disabled, whose earlier attempts were sent; with+ attempts, n8n timed out or was unreachable. An empty list means nothing has failed permanently+ yet: retries still in flight show only in the server logs, and an event that never matched the+ webhook leaves no row### Message Not Sendingdiff --git a/src/common/metrics/webhook-delivery-metrics.ts b/src/common/metrics/webhook-delivery-metrics.tsindex 7ac409f69..e547d2ca8 100644--- a/src/common/metrics/webhook-delivery-metrics.ts+++ b/src/common/metrics/webhook-delivery-metrics.ts@@ -1,6 +1,7 @@/*** Process-local monotonic counter of webhook delivery failures: deliveries whose every retry was- * exhausted, and deliveries never sent (shed, refused at shutdown, rejected before sending). It is+ * exhausted, deliveries never sent (shed, refused at shutdown, rejected before sending), and direct+ * deliveries stopped by shutdown between retries after earlier attempts were sent. It is* incremented once per row inserted into the durable `webhook_delivery_failures` dead-letter table;* a later replay may replace or remove that row, and the counter keeps the failure. Kept as a plain* in-process counter rather than a `COUNT(*)` over that table because the table is pruned on adiff --git a/src/common/utils/concurrency-limiter.spec.ts b/src/common/utils/concurrency-limiter.spec.tsindex 09729703e..5b89140c3 100644--- a/src/common/utils/concurrency-limiter.spec.ts+++ b/src/common/utils/concurrency-limiter.spec.ts@@ -67,6 +67,7 @@ describe('ConcurrencyLimiter', () => {it('clamps a non-positive max to 1 (never deadlocks)', async () => {const limiter = new ConcurrencyLimiter(0);+ expect(limiter.maxCount).toBe(1);await expect(limiter.run(() => Promise.resolve('ok'))).resolves.toBe('ok');});@@ -136,4 +137,110 @@ describe('ConcurrencyLimiter', () => {expect(limiter.activeCount).toBe(0);expect(limiter.queuedCount).toBe(0);});++ describe('yieldSlot', () => {+ const gate = (): { wait: Promise<void>; open: () => void } => {+ let open!: () => void;+ const wait = new Promise<void>(resolve => (open = resolve));+ return { wait, open };+ };++ it('lets a parked task run while the holder waits, then gives the holder a slot back', async () => {+ const limiter = new ConcurrencyLimiter(1);+ const order: string[] = [];+ let peak = 0;+ const sample = (): void => void (peak = Math.max(peak, limiter.activeCount));+ const sleep = gate();++ const a = limiter.run(async yieldSlot => {+ order.push('a:start');+ await yieldSlot(() => sleep.wait);+ sample();+ order.push('a:end');+ });+ const b = limiter.run(() => {+ sample();+ order.push('b');+ return Promise.resolve();+ });++ await b; // runs although A has not finished: A gave its slot up while waiting+ sleep.open();+ await a;++ expect(order).toEqual(['a:start', 'b', 'a:end']);+ expect(peak).toBe(1);+ expect(limiter.activeCount).toBe(0);+ });++ it('rejects the holder with "closed" when the limiter closes while it waits, releasing nothing twice', async () => {+ const limiter = new ConcurrencyLimiter(1);+ const sleep = gate();+ const a = limiter.run(yieldSlot => yieldSlot(() => sleep.wait));+ await Promise.resolve();++ limiter.close();+ sleep.open();++ await expect(a).rejects.toThrow('ConcurrencyLimiter closed');+ expect(limiter.activeCount).toBe(0);+ expect(limiter.queuedCount).toBe(0);+ });++ it('rejects a holder parked for its slot back when the limiter closes', async () => {+ const limiter = new ConcurrencyLimiter(1);+ const sleep = gate();+ const hold = gate();+ const a = limiter.run(yieldSlot => yieldSlot(() => sleep.wait));+ const b = limiter.run(() => hold.wait);+ sleep.open();+ await new Promise(resolve => setImmediate(resolve)); // A is now parked behind B++ expect(limiter.queuedCount).toBe(1);+ limiter.close();+ hold.open();++ await expect(a).rejects.toThrow('ConcurrencyLimiter closed');+ await expect(b).resolves.toBeUndefined();+ expect(limiter.activeCount).toBe(0);+ });++ it('takes the slot back even when the park queue is full', async () => {+ // The task was admitted already; shedding it mid-way would drop work that is half done.+ const limiter = new ConcurrencyLimiter(1, 0);+ const sleep = gate();+ const hold = gate();+ const a = limiter.run(async yieldSlot => {+ await yieldSlot(() => sleep.wait);+ return 'a';+ });+ const b = limiter.run(() => hold.wait);+ sleep.open();+ await new Promise(resolve => setImmediate(resolve));+ await expect(limiter.run(() => Promise.resolve())).rejects.toThrow('ConcurrencyLimiter queue full');++ hold.open();+ await expect(a).resolves.toBe('a');+ await b;+ expect(limiter.activeCount).toBe(0);+ });++ it('does not count a task taking its slot back against the park queue', async () => {+ const limiter = new ConcurrencyLimiter(1, 1);+ const sleep = gate();+ const hold = gate();+ const a = limiter.run(yieldSlot => yieldSlot(() => sleep.wait));+ const b = limiter.run(() => hold.wait);+ sleep.open();+ await new Promise(resolve => setImmediate(resolve)); // A is now parked behind B++ expect(limiter.reacquiringCount).toBe(1);+ const c = expect(limiter.run(() => Promise.resolve('c'))).resolves.toBe('c');++ hold.open();+ await Promise.all([a, b, c]);+ expect(limiter.reacquiringCount).toBe(0);+ expect(limiter.activeCount).toBe(0);+ });+ });});diff --git a/src/common/utils/concurrency-limiter.ts b/src/common/utils/concurrency-limiter.tsindex 2808e9b6f..5aab1f611 100644--- a/src/common/utils/concurrency-limiter.ts+++ b/src/common/utils/concurrency-limiter.ts@@ -7,6 +7,8 @@export class ConcurrencyLimiter {private active = 0;private readonly waiters: Array<(error?: Error) => void> = [];+ /** Waiters taking back a slot they gave up in yieldSlot: already admitted, so outside `maxQueued`. */+ private reacquiring = 0;private closed = false;/**@@ -24,6 +26,11 @@ export class ConcurrencyLimiter {this.maxQueued = Math.max(0, Math.floor(maxQueued));}+ /** Slots the limiter runs at once (after clamping). */+ get maxCount(): number {+ return this.max;+ }+/** Tasks currently holding a slot (running). */get activeCount(): number {return this.active;@@ -34,6 +41,11 @@ export class ConcurrencyLimiter {return this.waiters.length;}+ /** The part of {@link queuedCount} taking back a slot given up in yieldSlot (already admitted). */+ get reacquiringCount(): number {+ return this.reacquiring;+ }+/*** Stop admitting work: every parked task is rejected (its run() promise throws 'ConcurrencyLimiter* closed') and every future run() rejects the same way, so a shutdown path can drain instead of@@ -48,31 +60,71 @@ export class ConcurrencyLimiter {}}- async run<T>(task: () => Promise<T>): Promise<T> {+ /**+ * Run `task` once a slot is free. The task receives `yieldSlot(fn)`, which gives the slot up for+ * the duration of `fn` and takes one again before returning, so a task that has to wait (a retry+ * backoff) does not hold a slot it is not using. Taking the slot back is exempt from `maxQueued`+ * and does not count against it, since the task was already admitted, and rejects with+ * 'ConcurrencyLimiter closed' once the limiter is closed. Callers that never wait can ignore the argument.+ */+ async run<T>(task: (yieldSlot: <R>(fn: () => Promise<R>) => Promise<R>) => Promise<T>): Promise<T> {+ // Awaited only when parked: a free slot starts the task synchronously, as callers rely on.+ const parked = this.acquire(true);+ if (parked) await parked;+ let held = true;+ const yieldSlot = async <R>(fn: () => Promise<R>): Promise<R> => {+ if (!held) return fn();+ held = false;+ this.release();+ try {+ return await fn();+ } finally {+ const reparked = this.acquire(false);+ if (reparked) {+ this.reacquiring++;+ try {+ await reparked;+ } finally {+ this.reacquiring--;+ }+ }+ held = true;+ }+ };+ try {+ return await task(yieldSlot);+ } finally {+ // A failed re-acquire leaves nothing to release: releasing anyway would hand a waiter a slot+ // this task no longer holds, or push the count below zero.+ if (held) this.release();+ }+ }++ /** Take a free slot (returns undefined) or park for one (returns the promise to await). */+ private acquire(bounded: boolean): Promise<void> | undefined {if (this.closed) {throw new Error('ConcurrencyLimiter closed');}if (this.active < this.max) {this.active++;- } else {- if (this.waiters.length >= this.maxQueued) {- throw new Error('ConcurrencyLimiter queue full');- }- // Park until a finishing task HANDS us its slot (or close() rejects us). We must not increment- // on wake: the count was never released (it was transferred), so re-incrementing would- // over-admit when a fresh run() raced into the microtask gap and already took a (wrongly-freed)- // slot.- await new Promise<void>((resolve, reject) => this.waiters.push(error => (error ? reject(error) : resolve())));+ return undefined;}- try {- return await task();- } finally {- const next = this.waiters.shift();- if (next) {- next(); // transfer this slot to the next waiter — active count stays the same (no free window)- } else {- this.active--; // no one waiting: actually release the slot- }+ if (bounded && this.waiters.length - this.reacquiring >= this.maxQueued) {+ throw new Error('ConcurrencyLimiter queue full');+ }+ // Park until a finishing task HANDS us its slot (or close() rejects us). We must not increment+ // on wake: the count was never released (it was transferred), so re-incrementing would+ // over-admit when a fresh run() raced into the microtask gap and already took a (wrongly-freed)+ // slot.+ return new Promise<void>((resolve, reject) => this.waiters.push(error => (error ? reject(error) : resolve())));+ }++ private release(): void {+ const next = this.waiters.shift();+ if (next) {+ next(); // transfer this slot to the next waiter — active count stays the same (no free window)+ } else {+ this.active--; // no one waiting: actually release the slot}}}diff --git a/src/config/configuration.spec.ts b/src/config/configuration.spec.tsindex 7c0f1a119..283e70d66 100644--- a/src/config/configuration.spec.ts+++ b/src/config/configuration.spec.ts@@ -319,6 +319,27 @@ describe('configuration — webhook payload cap is fail-safe', () => {});});+describe('configuration: webhook degraded session concurrency', () => {+ const orig = process.env.WEBHOOK_DEGRADED_SESSION_CONCURRENCY;+ afterEach(() => {+ if (orig === undefined) delete process.env.WEBHOOK_DEGRADED_SESSION_CONCURRENCY;+ else process.env.WEBHOOK_DEGRADED_SESSION_CONCURRENCY = orig;+ });++ it('is undefined when unset or unusable, so each path derives it from its own pool', () => {+ for (const value of [undefined, '', 'abc', '0', '-2']) {+ if (value === undefined) delete process.env.WEBHOOK_DEGRADED_SESSION_CONCURRENCY;+ else process.env.WEBHOOK_DEGRADED_SESSION_CONCURRENCY = value;+ expect(configuration().webhook.degradedSessionConcurrency).toBeUndefined();+ }+ });++ it('honors a positive override', () => {+ process.env.WEBHOOK_DEGRADED_SESSION_CONCURRENCY = '3';+ expect(configuration().webhook.degradedSessionConcurrency).toBe(3);+ });+});+describe('configuration — webhook shutdown drain is fail-safe', () => {const orig = process.env.WEBHOOK_SHUTDOWN_DRAIN_MS;afterEach(() => {diff --git a/src/config/configuration.ts b/src/config/configuration.tsindex f854eeac8..b0732702d 100644--- a/src/config/configuration.ts+++ b/src/config/configuration.ts@@ -271,6 +271,13 @@ export default () => ({// Bound parked inline deliveries as well as active sockets. Queue-full dispatches are recorded in// webhook_delivery_failures instead of retaining payload closures without limit.dispatchMaxQueued: parseInt(process.env.WEBHOOK_DISPATCH_MAX_QUEUED || '1000', 10),+ // How many deliveries to failing receivers one session may run at once, per node, in direct and+ // queued mode alike; healthy receivers are not limited. Unset or garbage leaves it undefined, and+ // each path then uses a quarter of its own pool (at least 1).+ degradedSessionConcurrency: (() => {+ const n = parseInt(process.env.WEBHOOK_DEGRADED_SESSION_CONCURRENCY ?? '', 10);+ return Number.isInteger(n) && n > 0 ? n : undefined;+ })(),// Upper bound on the serialized webhook body after webhook:before hooks ran; oversize payloads// are recorded as undelivered instead of being sent/persisted. Default 1 MiB. Fail-safe like the// other byte caps: a non-numeric or non-positive value falls back to the default. 0 is NOT andiff --git a/src/config/env-precedence.ts b/src/config/env-precedence.tsindex 5c3b62f9a..d3309d4e3 100644--- a/src/config/env-precedence.ts+++ b/src/config/env-precedence.ts@@ -168,6 +168,7 @@ export const BLANK_SHADOWED_ENV_KEYS: string[] = ['WEBHOOK_MAX_PER_SESSION','WEBHOOK_FAILURE_RETENTION_DAYS','WEBHOOK_WORKER_CONCURRENCY',+ 'WEBHOOK_DEGRADED_SESSION_CONCURRENCY','INGRESS_INSTANCE_LIMIT','INGRESS_INSTANCE_TTL','INGRESS_IP_LIMIT',diff --git a/src/config/env.validation.spec.ts b/src/config/env.validation.spec.tsindex abea7c159..b874b16c1 100644--- a/src/config/env.validation.spec.ts+++ b/src/config/env.validation.spec.ts@@ -551,6 +551,7 @@ describe('validateEnv', () => {'S3_REPROBE_INTERVAL_MS','STORAGE_EXPORT_TTL_MS','STORAGE_EXPORT_SWEEP_MAX_AGE_MS',+ 'WEBHOOK_DEGRADED_SESSION_CONCURRENCY',])('rejects a unit-suffixed or non-positive %s and accepts a plain count', key => {expect(() => validateEnv({ [key]: '1h' })).toThrow(new RegExp(`${key} must be a positive integer`));expect(() => validateEnv({ [key]: '0' })).toThrow(new RegExp(`${key} must be a positive integer`));diff --git a/src/config/env.validation.ts b/src/config/env.validation.tsindex e140f0432..30f0fdf6e 100644--- a/src/config/env.validation.ts+++ b/src/config/env.validation.ts@@ -307,6 +307,7 @@ export function validateEnv(config: EnvConfig): EnvConfig {'HEADERS_TIMEOUT_MS','KEEPALIVE_TIMEOUT_MS','WEBHOOK_DISPATCH_CONCURRENCY',+ 'WEBHOOK_DEGRADED_SESSION_CONCURRENCY',// 0 would reject every webhook dispatch (a total, silent webhook outage).'WEBHOOK_MAX_PAYLOAD_BYTES',// 0 would refuse every request carrying a body (a self-DoS), so the budget is positive-only.diff --git a/src/modules/metrics/metrics.service.ts b/src/modules/metrics/metrics.service.tsindex 5280cdc7e..ce16907d3 100644--- a/src/modules/metrics/metrics.service.ts+++ b/src/modules/metrics/metrics.service.ts@@ -216,7 +216,7 @@ export class MetricsService implements OnModuleDestroy {}lines.push(- '# HELP openwa_webhook_delivery_failures_total Webhook delivery failures recorded since process start: retries exhausted, or never sent (shed, refused at shutdown, rejected before sending).',+ '# HELP openwa_webhook_delivery_failures_total Webhook delivery failures recorded since process start: retries exhausted, never sent (shed, refused at shutdown, rejected before sending), or stopped by shutdown between direct retries.',);lines.push('# TYPE openwa_webhook_delivery_failures_total counter');lines.push(`openwa_webhook_delivery_failures_total ${getWebhookDeliveryFailuresTotal()}`);diff --git a/src/modules/queue/processors/webhook.processor.spec.ts b/src/modules/queue/processors/webhook.processor.spec.tsindex a200bcc63..619ee912c 100644--- a/src/modules/queue/processors/webhook.processor.spec.ts+++ b/src/modules/queue/processors/webhook.processor.spec.ts@@ -1,4 +1,4 @@-import { Job } from 'bullmq';+import { DelayedError, Job } from 'bullmq';import { FindOperator, Repository } from 'typeorm';import { ConfigService } from '@nestjs/config';import { WebhookProcessor } from './webhook.processor';@@ -469,4 +469,131 @@ describe('WebhookProcessor', () => {expect(hookManager.execute).not.toHaveBeenCalled();});});++ describe('failing-receiver gate', () => {+ // One slot per session for failing receivers, so the arithmetic below stays small.+ const gatedProcessor = (retryDelay = 1000): WebhookProcessor => {+ configService.get.mockImplementation((key: string, def?: unknown) => {+ if (key === 'webhook.degradedSessionConcurrency') return 1;+ if (key === 'webhook.retryDelay') return retryDelay;+ return key === 'webhook.timeout' ? 25000 : def;+ });+ return new WebhookProcessor(+ repo as unknown as Repository<Webhook>,+ failureRepo as unknown as Repository<WebhookDeliveryFailure>,+ hookManager as unknown as HookManager,+ configService as unknown as ConfigService,+ );+ };+ const withMoveToDelayed = (job: Job<WebhookJobData>): jest.Mock => {+ const move = jest.fn().mockResolvedValue(undefined);+ (job as unknown as { moveToDelayed: jest.Mock }).moveToDelayed = move;+ return move;+ };+ const hang = (): { release: () => void } => {+ const handle = { release: () => undefined as void };+ mockFetch.mockImplementationOnce(+ () => new Promise(resolve => (handle.release = () => resolve({ ok: true, status: 200 }))),+ );+ return handle;+ };++ it('delays a job for a failing webhook past the session cap without spending an attempt or failing it', async () => {+ const gp = gatedProcessor();+ mockFetch.mockResolvedValueOnce({ ok: false, status: 500, statusText: 'Server Error' });+ await expect(gp.process(makeJob())).rejects.toThrow('HTTP 500');+ const first = hang();+ mockFetch.mockClear();+ const inFlight = gp.process(makeJob(), 'tok-1');+ await new Promise(resolve => setImmediate(resolve));+ repo.findOne.mockClear();+ failureRepo.insert.mockClear();++ // The last attempt of its job: a gate bounce must still not dead-letter it.+ const job = makeJob({}, 2);+ const move = withMoveToDelayed(job);+ const before = Date.now();+ await expect(gp.process(job, 'tok-2')).rejects.toBeInstanceOf(DelayedError);++ expect(move).toHaveBeenCalledWith(expect.any(Number), 'tok-2');+ expect((move.mock.calls[0] as [number])[0]).toBeGreaterThanOrEqual(before + 1000);+ expect(repo.findOne).not.toHaveBeenCalled();+ expect(mockFetch).toHaveBeenCalledTimes(1); // only the in-flight job's POST+ expect(failureRepo.insert).not.toHaveBeenCalled();++ first.release();+ await inFlight;+ });++ // BullMQ promotes a job delayed to "now" straight back to wait, so a zero delay would spin it.+ it('waits at least a second before a bounced job returns, even with WEBHOOK_RETRY_DELAY=0', async () => {+ const gp = gatedProcessor(0);+ mockFetch.mockResolvedValueOnce({ ok: false, status: 500, statusText: 'Server Error' });+ await expect(gp.process(makeJob())).rejects.toThrow('HTTP 500');+ const first = hang();+ const inFlight = gp.process(makeJob(), 'tok-1');+ await new Promise(resolve => setImmediate(resolve));++ const job = makeJob();+ const move = withMoveToDelayed(job);+ const before = Date.now();+ await expect(gp.process(job, 'tok-2')).rejects.toBeInstanceOf(DelayedError);++ expect((move.mock.calls[0] as [number])[0]).toBeGreaterThanOrEqual(before + 1000);+ first.release();+ await inFlight;+ });++ it('never holds back a healthy webhook, however many of its jobs run at once', async () => {+ const gp = gatedProcessor();+ const a = hang();+ const b = hang();+ const jobs = [gp.process(makeJob()), gp.process(makeJob())];+ await new Promise(resolve => setImmediate(resolve));++ expect(mockFetch).toHaveBeenCalledTimes(2);+ a.release();+ b.release();+ await Promise.all(jobs);+ });++ it('lifts the gate when a job finds its failing webhook gone', async () => {+ const gp = gatedProcessor();+ mockFetch.mockResolvedValueOnce({ ok: false, status: 500, statusText: 'Server Error' });+ await expect(gp.process(makeJob())).rejects.toThrow('HTTP 500');+ repo.findOne.mockResolvedValueOnce(null);+ await expect(gp.process(makeJob())).resolves.toMatchObject({ success: false });++ // Re-created or re-enabled under the same id: its jobs are no longer held to one slot.+ const a = hang();+ const b = hang();+ const jobs = [gp.process(makeJob()), gp.process(makeJob())];+ await new Promise(resolve => setImmediate(resolve));+ expect(mockFetch).toHaveBeenCalledTimes(3);+ a.release();+ b.release();+ await Promise.all(jobs);+ });++ it('lifts the gate once the webhook answers 2xx again, and frees the slot on success and on failure', async () => {+ const gp = gatedProcessor();+ mockFetch.mockResolvedValueOnce({ ok: false, status: 500, statusText: 'Server Error' });+ await expect(gp.process(makeJob())).rejects.toThrow('HTTP 500');+ // Gated now, with a free slot: a failing job takes it and gives it back when it throws.+ mockFetch.mockResolvedValueOnce({ ok: false, status: 500, statusText: 'Server Error' });+ await expect(gp.process(makeJob())).rejects.toThrow('HTTP 500');+ // ...and a succeeding job takes it, gives it back, and clears the failing mark.+ mockFetch.mockResolvedValueOnce({ ok: true, status: 200 });+ await expect(gp.process(makeJob())).resolves.toMatchObject({ success: true });++ const a = hang();+ const b = hang();+ const jobs = [gp.process(makeJob()), gp.process(makeJob())];+ await new Promise(resolve => setImmediate(resolve));+ expect(mockFetch).toHaveBeenCalledTimes(5);+ a.release();+ b.release();+ await Promise.all(jobs);+ });+ });});diff --git a/src/modules/queue/processors/webhook.processor.ts b/src/modules/queue/processors/webhook.processor.tsindex 9505aad9d..5248b503e 100644--- a/src/modules/queue/processors/webhook.processor.ts+++ b/src/modules/queue/processors/webhook.processor.ts@@ -1,5 +1,5 @@import { Processor, WorkerHost, OnWorkerEvent } from '@nestjs/bullmq';-import { Job } from 'bullmq';+import { DelayedError, Job } from 'bullmq';import { InjectRepository } from '@nestjs/typeorm';import { ConfigService } from '@nestjs/config';import { Repository } from 'typeorm';@@ -54,6 +54,15 @@ interface WebhookDeliveryContext {@Processor(QUEUE_NAMES.WEBHOOK, { connection: workerConnectionOptions(), concurrency: webhookWorkerConcurrency() })export class WebhookProcessor extends WorkerHost {private readonly logger = createLogger('WebhookProcessor');+ /**+ * Webhooks whose last attempt on this worker failed, until one succeeds. A job for one of them+ * takes a per-session slot first, at most `degradedSessionConcurrency` per session; over that it+ * goes back to the delayed set without spending an attempt. One session's failing receivers then+ * cannot fill the shared pool, while healthy receivers are never held back.+ */+ private readonly failingWebhooks = new Set<string>();+ private readonly degradedInFlight = new Map<string, number>();+ private readonly degradedSessionConcurrency: number;constructor(@InjectRepository(Webhook, 'data')@@ -64,9 +73,40 @@ export class WebhookProcessor extends WorkerHost {private readonly configService: ConfigService,) {super();+ // WEBHOOK_DEGRADED_SESSION_CONCURRENCY, else a quarter of the worker pool.+ this.degradedSessionConcurrency =+ this.configService.get<number | undefined>('webhook.degradedSessionConcurrency') ??+ Math.max(1, Math.floor(webhookWorkerConcurrency() / 4));+ }++ async process(job: Job<WebhookJobData>, token?: string): Promise<WebhookJobResult> {+ const { webhookId, payload } = job.data;+ const sessionId = payload.sessionId;+ const gated = this.failingWebhooks.has(webhookId);+ if (gated) {+ const inFlight = this.degradedInFlight.get(sessionId) ?? 0;+ if (inFlight >= this.degradedSessionConcurrency) {+ // Before the try below on purpose: this is not an attempt, so it must never be logged as a+ // failure or file a dead-letter row. moveToDelayed does not spend one of the job's attempts.+ // At least a second: a job delayed to "now" is promoted straight back and would spin.+ const delay = Math.max(1000, this.configService.get<number>('webhook.retryDelay', 5000));+ await job.moveToDelayed(Date.now() + delay + Math.floor(Math.random() * delay), token);+ throw new DelayedError();+ }+ this.degradedInFlight.set(sessionId, inFlight + 1);+ }+ try {+ return await this.deliver(job);+ } finally {+ if (gated) {+ const left = (this.degradedInFlight.get(sessionId) ?? 1) - 1;+ if (left > 0) this.degradedInFlight.set(sessionId, left);+ else this.degradedInFlight.delete(sessionId);+ }+ }}- async process(job: Job<WebhookJobData>): Promise<WebhookJobResult> {+ private async deliver(job: Job<WebhookJobData>): Promise<WebhookJobResult> {const { webhookId, event, payload, maxRetries } = job.data;const startTime = Date.now();const sessionId = payload.sessionId;@@ -109,6 +149,7 @@ export class WebhookProcessor extends WorkerHost {idempotencyKey: payload.idempotencyKey,action: 'webhook_skipped_stale',});+ this.failingWebhooks.delete(webhookId);return { statusCode: 0, success: false, error: 'webhook removed, disabled or unsubscribed', responseTime: 0 };}ctx.url = current.url;@@ -123,6 +164,7 @@ export class WebhookProcessor extends WorkerHost {);const { status, responseTime } = await this.postToReceiver(ctx, body, requestHeaders);+ this.failingWebhooks.delete(webhookId);await this.recordSuccessfulDelivery(ctx, status, responseTime);return {statusCode: status,@@ -130,6 +172,7 @@ export class WebhookProcessor extends WorkerHost {responseTime,};} catch (error) {+ this.failingWebhooks.add(webhookId);await this.recordDeliveryFailure(ctx, error);// Re-throw to trigger BullMQ retrythrow error;diff --git a/src/modules/queue/redis-connection.ts b/src/modules/queue/redis-connection.tsindex cd216a6bc..3ad2d6782 100644--- a/src/modules/queue/redis-connection.ts+++ b/src/modules/queue/redis-connection.ts@@ -29,13 +29,14 @@ const DEFAULT_WEBHOOK_WORKER_CONCURRENCY = 10;* Webhook Worker concurrency. BullMQ defaults a Worker to 1, which serializes ALL webhook deliveries* process-wide: one slow or timing-out receiver head-of-line-blocks every other session's webhooks* until it finishes (up to WEBHOOK_TIMEOUT + retries). Running several in parallel lets healthy- * receivers proceed while a few slots wait on a stuck one, but the pool is shared and not isolated- * per webhook: each attempt against a dead receiver holds a slot for up to WEBHOOK_TIMEOUT, and its- * retries re-enter the same pool. Once that receiver's events per second x attempts x- * WEBHOOK_TIMEOUT (in seconds) reach the concurrency, every slot is busy with it and all other- * webhooks wait. Size the value above that product. Override via WEBHOOK_WORKER_CONCURRENCY; a- * non-positive/garbage value falls back to the default. (Read at module import like- * workerConnectionOptions above.)+ * receivers proceed while a few slots wait on a stuck one. The pool is shared, and each attempt+ * against a dead receiver holds a slot for up to WEBHOOK_TIMEOUT, so the processor caps how many+ * jobs to failing receivers one session starts at once (WEBHOOK_DEGRADED_SESSION_CONCURRENCY,+ * default a quarter of this value; a job already running when the failure lands is not counted)+ * and returns the rest to the delayed set. Size this value above that cap times the number of+ * sessions whose receivers may fail at once, or those sessions fill the pool. Override via+ * WEBHOOK_WORKER_CONCURRENCY; a non-positive/garbage value falls back to the default. (Read at+ * module import like workerConnectionOptions above.)*/export function webhookWorkerConcurrency(): number {return resolveNonNegativeIntEnv(process.env.WEBHOOK_WORKER_CONCURRENCY, 0) || DEFAULT_WEBHOOK_WORKER_CONCURRENCY;diff --git a/src/modules/webhook/webhook-delivery.service.spec.ts b/src/modules/webhook/webhook-delivery.service.spec.tsindex eb746d113..07e8bf093 100644--- a/src/modules/webhook/webhook-delivery.service.spec.ts+++ b/src/modules/webhook/webhook-delivery.service.spec.ts@@ -1747,6 +1747,47 @@ describe('WebhookDeliveryService', () => {expect(failureRepository.insert).not.toHaveBeenCalled();});+ it('frees the dispatch slot while a queue-fallback delivery waits to retry', async () => {+ const queueService = await buildQueueService(<T>(key: string, def?: T): T | boolean | number => {+ if (key === 'queue.enabled') return true;+ if (key === 'webhook.retryDelay') return 50;+ return def as T;+ });+ (queueService as unknown as { dispatchLimiter: ConcurrencyLimiter }).dispatchLimiter = new ConcurrencyLimiter(+ 1,+ 10,+ );+ const wA = createMockWebhook({ id: 'wh-a', sessionId: 'sess-a', url: 'https://a.example/hook', retryCount: 2 });+ const wB = createMockWebhook({ id: 'wh-b', sessionId: 'sess-b', url: 'https://b.example/hook' });+ (repository.find as jest.Mock).mockImplementation((opts: { where: { sessionId: string } }) =>+ Promise.resolve(opts.where.sessionId === 'sess-a' ? [wA] : [wB]),+ );+ (repository.findOne as jest.Mock).mockResolvedValue(wA);+ (hookManager.execute as jest.Mock).mockResolvedValue({ continue: true, data: {} });+ const order: string[] = [];+ webhookQueue.add.mockImplementation((name: string) => {+ order.push(`add:${name}`);+ return name === 'webhook-wh-a' ? Promise.reject(new Error('redis down')) : Promise.resolve(undefined);+ });+ const mockFetch = undiciFetch as jest.Mock;+ mockFetch.mockImplementation((url: string) => {+ order.push(`post:${url}`);+ return Promise.reject(new Error('receiver down'));+ });++ const a = queueService.dispatch('sess-a', 'message.received', {});+ for (let i = 0; i < 50 && mockFetch.mock.calls.length === 0; i++) await new Promise(r => setImmediate(r));+ await queueService.dispatch('sess-b', 'message.received', {});+ await a;++ expect(order).toEqual([+ 'add:webhook-wh-a',+ 'post:https://a.example/hook',+ 'add:webhook-wh-b',+ 'post:https://a.example/hook',+ ]);+ });+it('falls back to direct delivery when queue add fails', async () => {const queueService = await buildQueueService(<T>(key: string, def?: T): T | boolean | number => {if (key === 'queue.enabled') return true;@@ -1819,6 +1860,8 @@ describe('WebhookDeliveryService', () => {const hooks = (hookManager.execute as jest.Mock).mock.calls.map(([name]) => name as string);expect(hooks).not.toContain('webhook:delivered');expect(hooks).not.toContain('webhook:after');+ // The failed first attempt marked it failing; a webhook that is gone must not stay gated.+ expect((service as unknown as { failingWebhooks: Set<string> }).failingWebhooks.has(webhook.id)).toBe(false);});it('sends a retry to the current url, signed with the current secret', async () => {@@ -1874,6 +1917,442 @@ describe('WebhookDeliveryService', () => {});});+ describe('dispatch slot isolation (direct mode)', () => {+ const mockFetch = undiciFetch as jest.Mock;+ type Internals = {+ dispatchLimiter: ConcurrencyLimiter;+ dispatchMaxQueued: number;+ degradedSessionConcurrency: number;+ degradedSessionGates: Map<string, ConcurrencyLimiter>;+ failingWebhooks: Set<string>;+ };+ const internals = (): Internals => service as unknown as Internals;+ const hooksBySession: Record<string, Webhook[]> = {};+ const urlsCalled = (): string[] => (mockFetch.mock.calls as Array<[string]>).map(([url]) => url);+ const settle = async (until: () => boolean): Promise<void> => {+ for (let i = 0; i < 50 && !until(); i++) await new Promise(r => setImmediate(r));+ };++ beforeEach(() => {+ for (const key of Object.keys(hooksBySession)) delete hooksBySession[key];+ (repository.find as jest.Mock).mockImplementation((opts: { where: { sessionId: string } }) =>+ Promise.resolve(hooksBySession[opts.where.sessionId] ?? []),+ );+ (repository.findOne as jest.Mock).mockImplementation((opts: { where: { id: string } }) =>+ Promise.resolve(+ Object.values(hooksBySession)+ .flat()+ .find(w => w.id === opts.where.id) ?? null,+ ),+ );+ (hookManager.execute as jest.Mock).mockResolvedValue({ continue: true, data: {} });+ mockFetch.mockReset();+ });++ afterEach(() => mockFetch.mockReset());++ // A failing receiver's backoff held its slot for the whole retry loop, so a few of them stalled+ // every other webhook of every session on the node.+ it('frees the dispatch slot while a failed delivery waits to retry', async () => {+ internals().dispatchLimiter = new ConcurrencyLimiter(1, 1000);+ hooksBySession['sess-a'] = [createMockWebhook({ id: 'wh-a', url: 'https://a.example/hook', retryCount: 2 })];+ hooksBySession['sess-b'] = [createMockWebhook({ id: 'wh-b', url: 'https://b.example/hook', retryCount: 1 })];+ mockFetch.mockImplementation((url: string) =>+ url.startsWith('https://a.')+ ? Promise.reject(new Error('receiver down'))+ : Promise.resolve({ ok: true, status: 200 }),+ );++ const a = service.dispatch('sess-a', 'message.received', {});+ await settle(() => mockFetch.mock.calls.length > 0);+ await service.dispatch('sess-b', 'message.received', {});+ await a;++ expect(urlsCalled()).toEqual(['https://a.example/hook', 'https://b.example/hook', 'https://a.example/hook']);+ expect(internals().dispatchLimiter.activeCount).toBe(0);+ });++ it('records a delivery woken from its backoff by shutdown as unsent and keeps its outbox row', async () => {+ internals().dispatchLimiter = new ConcurrencyLimiter(1, 1000);+ hooksBySession['sess-a'] = [createMockWebhook({ id: 'wh-a', url: 'https://a.example/hook', retryCount: 3 })];+ mockFetch.mockRejectedValue(new Error('receiver down'));++ const a = service.dispatch('sess-a', 'message.received', {});+ await settle(() => mockFetch.mock.calls.length > 0);+ await service.onModuleDestroy();+ await a;++ // Not a terminal failure: the retry budget was not spent, so the next start replays it.+ expect(mockFetch).toHaveBeenCalledTimes(1);+ expect(insertedFailures).toEqual([+ expect.objectContaining({ webhookId: 'wh-a', attempts: 0, lastError: 'ConcurrencyLimiter closed' }),+ ]);+ expect(outboxService.close).not.toHaveBeenCalled();+ });++ // Asleep in its backoff a delivery holds no slot, so only the in-flight count keeps shutdown+ // from returning before it wakes and records itself.+ it('waits on shutdown for a delivery asleep in its retry backoff', async () => {+ internals().dispatchLimiter = new ConcurrencyLimiter(1, 1000);+ hooksBySession['sess-a'] = [createMockWebhook({ id: 'wh-a', url: 'https://a.example/hook', retryCount: 3 })];+ mockFetch.mockRejectedValue(new Error('receiver down'));+ const backoff = sleep as unknown as jest.Mock;+ backoff.mockClear();+ let wake: () => void = () => undefined;+ backoff.mockImplementationOnce(() => new Promise<void>(r => (wake = r)));+ const order: string[] = [];+ (failureRepository.insert as jest.Mock).mockImplementation((row: Partial<WebhookDeliveryFailure>) => {+ insertedFailures.push(row);+ order.push('insert');+ return Promise.resolve({});+ });++ const a = service.dispatch('sess-a', 'message.received', {});+ await settle(() => backoff.mock.calls.length > 0);+ expect(internals().dispatchLimiter.activeCount).toBe(0);+ const destroyed = service.onModuleDestroy().then(() => order.push('destroyed'));+ await settle(() => order.length > 0);+ expect(order).toEqual([]);++ wake();+ await destroyed;+ await a;+ expect(order).toEqual(['insert', 'destroyed']);+ });++ it("holds one session's failing receivers to their share of the pool", async () => {+ internals().dispatchLimiter = new ConcurrencyLimiter(2, 1000);+ internals().degradedSessionConcurrency = 1;+ internals().failingWebhooks.add('wh-a1').add('wh-a2');+ hooksBySession['sess-a'] = [+ createMockWebhook({ id: 'wh-a1', url: 'https://a1.example/hook', retryCount: 1 }),+ createMockWebhook({ id: 'wh-a2', url: 'https://a2.example/hook', retryCount: 1 }),+ ];+ hooksBySession['sess-b'] = [createMockWebhook({ id: 'wh-b', url: 'https://b.example/hook', retryCount: 1 })];+ const hanging: Array<() => void> = [];+ mockFetch.mockImplementation((url: string) =>+ url.startsWith('https://b.')+ ? Promise.resolve({ ok: true, status: 200 })+ : new Promise(resolve => hanging.push(() => resolve({ ok: true, status: 200 }))),+ );++ const a = service.dispatch('sess-a', 'message.received', {});+ await settle(() => mockFetch.mock.calls.length > 0);+ await service.dispatch('sess-b', 'message.received', {});++ expect(urlsCalled().filter(u => !u.startsWith('https://b.'))).toHaveLength(1);+ expect(urlsCalled()).toContain('https://b.example/hook');++ hanging.shift()!();+ await settle(() => hanging.length > 0);+ hanging.shift()!();+ await a;+ // A 2xx lifts the gate for the webhook that answered, and the idle session gate is dropped.+ expect(internals().failingWebhooks.size).toBe(0);+ expect(internals().degradedSessionGates.size).toBe(0);+ });++ // A parked delivery can wait behind the gate for minutes, so the row it was dispatched with is+ // stale by the time the gate admits it: the operator may have disabled or re-pointed the webhook.+ it('re-reads the webhook when the session gate admits a parked delivery', async () => {+ internals().dispatchLimiter = new ConcurrencyLimiter(2, 1000);+ internals().degradedSessionConcurrency = 1;+ internals().failingWebhooks.add('wh-a1').add('wh-a2').add('wh-a3');+ hooksBySession['sess-a'] = [+ createMockWebhook({ id: 'wh-a1', url: 'https://a1.example/hook', retryCount: 1 }),+ createMockWebhook({ id: 'wh-a2', url: 'https://a2.example/hook', retryCount: 1 }),+ createMockWebhook({ id: 'wh-a3', url: 'https://a3.example/hook', retryCount: 1 }),+ ];+ const hanging: Array<() => void> = [];+ mockFetch.mockImplementation(+ () => new Promise(resolve => hanging.push(() => resolve({ ok: true, status: 200 }))),+ );++ const a = service.dispatch('sess-a', 'message.received', {});+ await settle(() => mockFetch.mock.calls.length > 0);+ expect(urlsCalled()).toEqual(['https://a1.example/hook']);+ // While wh-a2 and wh-a3 are parked, one is disabled and the other re-pointed.+ hooksBySession['sess-a'] = [+ hooksBySession['sess-a'][0],+ createMockWebhook({ id: 'wh-a2', url: 'https://a2.example/hook', retryCount: 1, active: false }),+ createMockWebhook({ id: 'wh-a3', url: 'https://a3-new.example/hook', retryCount: 1 }),+ ];+ mockFetch.mockResolvedValue({ ok: true, status: 200 });++ hanging.shift()!();+ await a;++ expect(urlsCalled()).toEqual(['https://a1.example/hook', 'https://a3-new.example/hook']);+ expect(insertedFailures).toEqual([]);+ expect((outboxService.close.mock.calls as string[][]).map(c => c[0]).sort()).toEqual(['wh-a1', 'wh-a2', 'wh-a3']);+ expect(internals().failingWebhooks.size).toBe(0);+ expect(internals().degradedSessionGates.size).toBe(0);+ });++ it('delivers with the dispatch-time row when the parked re-read fails', async () => {+ internals().failingWebhooks.add('wh-a');+ hooksBySession['sess-a'] = [createMockWebhook({ id: 'wh-a', url: 'https://a.example/hook', retryCount: 1 })];+ (repository.findOne as jest.Mock).mockRejectedValueOnce(new Error('db busy'));+ mockFetch.mockResolvedValue({ ok: true, status: 200 });+ const warn = jest.spyOn((service as unknown as { logger: { warn: jest.Mock } }).logger, 'warn');++ await service.dispatch('sess-a', 'message.received', {});++ expect(warn).toHaveBeenCalledWith(+ expect.any(String),+ expect.objectContaining({ webhookId: 'wh-a', error: 'db busy', action: 'webhook_stale_check_failed' }),+ );+ expect(urlsCalled()).toEqual(['https://a.example/hook']);+ expect((outboxService.close.mock.calls as string[][]).map(c => c[0])).toEqual(['wh-a']);+ expect(internals().failingWebhooks.size).toBe(0);+ });++ it('gives healthy receivers the whole pool', async () => {+ internals().dispatchLimiter = new ConcurrencyLimiter(2, 1000);+ internals().degradedSessionConcurrency = 1;+ hooksBySession['sess-a'] = [+ createMockWebhook({ id: 'wh-a1', url: 'https://a1.example/hook', retryCount: 1 }),+ createMockWebhook({ id: 'wh-a2', url: 'https://a2.example/hook', retryCount: 1 }),+ ];+ const hanging: Array<() => void> = [];+ mockFetch.mockImplementation(+ () => new Promise(resolve => hanging.push(() => resolve({ ok: true, status: 200 }))),+ );++ const a = service.dispatch('sess-a', 'message.received', {});+ await settle(() => mockFetch.mock.calls.length > 1);++ expect(mockFetch).toHaveBeenCalledTimes(2);+ hanging.forEach(release => release());+ await a;+ });++ it('marks a webhook failing on a failed attempt', async () => {+ const webhook = createMockWebhook({ id: 'wh-a', retryCount: 1 });+ mockFetch.mockRejectedValue(new Error('receiver down'));++ await service.redeliver(webhook, 'sess-1', 'message.received', 'k', {});++ expect(internals().failingWebhooks.has('wh-a')).toBe(true);+ });++ it('sheds a delivery that would park past the dispatch queue bound behind the session gate', async () => {+ internals().dispatchLimiter = new ConcurrencyLimiter(2, 0);+ internals().dispatchMaxQueued = 0;+ internals().degradedSessionConcurrency = 1;+ internals().failingWebhooks.add('wh-a1').add('wh-a2');+ hooksBySession['sess-a'] = [+ createMockWebhook({ id: 'wh-a1', url: 'https://a1.example/hook', retryCount: 1 }),+ createMockWebhook({ id: 'wh-a2', url: 'https://a2.example/hook', retryCount: 1 }),+ ];+ const hanging: Array<() => void> = [];+ mockFetch.mockImplementation(+ () => new Promise(resolve => hanging.push(() => resolve({ ok: true, status: 200 }))),+ );++ const a = service.dispatch('sess-a', 'message.received', {});+ await settle(() => insertedFailures.length > 0);++ expect(insertedFailures).toEqual([+ expect.objectContaining({ webhookId: 'wh-a2', attempts: 0, lastError: 'ConcurrencyLimiter queue full' }),+ ]);+ hanging.forEach(release => release());+ await a;+ expect((outboxService.close.mock.calls as string[][]).map(c => c[0])).toEqual(['wh-a1']);+ expect(internals().degradedSessionGates.size).toBe(0);+ for (const [row] of outboxService.open.mock.calls as Array<[{ idempotencyKey: string }]>) {+ expect(service.isLocallyPending(row.idempotencyKey)).toBe(false);+ }+ });++ it('sheds a healthy delivery once parked deliveries reach the dispatch queue bound', async () => {+ internals().dispatchLimiter = new ConcurrencyLimiter(1, 1000);+ internals().dispatchMaxQueued = 1;+ internals().degradedSessionConcurrency = 1;+ internals().failingWebhooks.add('wh-a1').add('wh-a2');+ hooksBySession['sess-a'] = [+ createMockWebhook({ id: 'wh-a1', url: 'https://a1.example/hook', retryCount: 1 }),+ createMockWebhook({ id: 'wh-a2', url: 'https://a2.example/hook', retryCount: 1 }),+ ];+ hooksBySession['sess-b'] = [createMockWebhook({ id: 'wh-b', url: 'https://b.example/hook', retryCount: 1 })];+ const hanging: Array<() => void> = [];+ mockFetch.mockImplementation(+ () => new Promise(resolve => hanging.push(() => resolve({ ok: true, status: 200 }))),+ );++ // wh-a1 holds the only slot and wh-a2 parks behind the session gate: the queue bound is spent.+ const a = service.dispatch('sess-a', 'message.received', {});+ await settle(() => mockFetch.mock.calls.length > 0);+ const b = service.dispatch('sess-b', 'message.received', {});+ await settle(() => insertedFailures.length > 0);++ expect(insertedFailures).toEqual([+ expect.objectContaining({ webhookId: 'wh-b', attempts: 0, lastError: 'ConcurrencyLimiter queue full' }),+ ]);+ await b;+ expect(outboxService.close).not.toHaveBeenCalled();+ while (hanging.length) {+ hanging.shift()!();+ await settle(() => hanging.length > 0 || outboxService.close.mock.calls.length > 1);+ }+ await a;+ });++ // Without a park cap of its own, one failing session could park up to the whole shared bound and+ // push every other session's deliveries to the shed path as soon as they needed to wait.+ it("sheds a failing session's own excess before it spends the shared park budget", async () => {+ internals().dispatchLimiter = new ConcurrencyLimiter(2, 4);+ internals().dispatchMaxQueued = 4;+ internals().degradedSessionConcurrency = 1;+ internals().failingWebhooks.add('wh-a1').add('wh-a2').add('wh-a3');+ hooksBySession['sess-a'] = ['a1', 'a2', 'a3'].map(n =>+ createMockWebhook({ id: `wh-${n}`, url: `https://${n}.example/hook`, retryCount: 1 }),+ );+ hooksBySession['sess-b'] = ['b1', 'b2'].map(n =>+ createMockWebhook({ id: `wh-${n}`, url: `https://${n}.example/hook`, retryCount: 1 }),+ );+ const hanging: Array<() => void> = [];+ mockFetch.mockImplementation(+ () => new Promise(resolve => hanging.push(() => resolve({ ok: true, status: 200 }))),+ );++ // wh-a1 runs, wh-a2 parks behind the session gate, and wh-a3 is past that gate's own cap.+ const a = service.dispatch('sess-a', 'message.received', {});+ await settle(() => insertedFailures.length > 0);+ expect(insertedFailures).toEqual([+ expect.objectContaining({ webhookId: 'wh-a3', attempts: 0, lastError: 'ConcurrencyLimiter queue full' }),+ ]);++ // The healthy session still finds room to park.+ const b = service.dispatch('sess-b', 'message.received', {});+ await settle(() => mockFetch.mock.calls.length > 1);+ expect(internals().dispatchLimiter.queuedCount).toBe(1);+ while (hanging.length) {+ hanging.shift()!();+ await settle(() => hanging.length > 0);+ }+ await Promise.all([a, b]);++ expect(insertedFailures).toHaveLength(1);+ expect(urlsCalled()).toEqual(expect.arrayContaining(['https://b1.example/hook', 'https://b2.example/hook']));+ });++ // A retry backoff gives its slot up, so without a count of the sleepers every failed attempt let+ // another delivery in, and the number held in heap grew with the event rate.+ it('counts deliveries waiting out a backoff against the dispatch bound', async () => {+ internals().dispatchLimiter = new ConcurrencyLimiter(1, 1000);+ internals().dispatchMaxQueued = 1;+ hooksBySession['sess-a'] = [createMockWebhook({ id: 'wh-a', url: 'https://a.example/hook', retryCount: 2 })];+ hooksBySession['sess-b'] = [createMockWebhook({ id: 'wh-b', url: 'https://b.example/hook', retryCount: 1 })];+ hooksBySession['sess-c'] = [createMockWebhook({ id: 'wh-c', url: 'https://c.example/hook', retryCount: 1 })];+ let wake: () => void = () => undefined;+ (sleep as unknown as jest.Mock).mockImplementationOnce(() => new Promise<void>(r => (wake = r)));+ let releaseB: () => void = () => undefined;+ mockFetch.mockImplementation((url: string) =>+ url.startsWith('https://a.')+ ? Promise.reject(new Error('receiver down'))+ : url.startsWith('https://b.')+ ? new Promise(resolve => (releaseB = () => resolve({ ok: true, status: 200 })))+ : Promise.resolve({ ok: true, status: 200 }),+ );++ // wh-a sleeps in its backoff without a slot, wh-b holds the slot: one running, one waiting.+ const a = service.dispatch('sess-a', 'message.received', {});+ await settle(() => (sleep as unknown as jest.Mock).mock.calls.length > 0);+ const b = service.dispatch('sess-b', 'message.received', {});+ await settle(() => mockFetch.mock.calls.length > 1);+ const c = service.dispatch('sess-c', 'message.received', {});+ await settle(() => insertedFailures.length > 0);++ expect(insertedFailures).toEqual([+ expect.objectContaining({ webhookId: 'wh-c', attempts: 0, lastError: 'ConcurrencyLimiter queue full' }),+ ]);+ await c;+ releaseB();+ await b;+ wake();+ await a;+ expect(urlsCalled()).not.toContain('https://c.example/hook');+ expect((outboxService.close.mock.calls as string[][]).map(call => call[0]).sort()).toEqual(['wh-a', 'wh-b']);+ });++ // A delivery woken from its backoff parks to take its slot back while it is still counted as+ // admitted, so counting the parked tasks as well shed deliveries below the bound.+ it('counts a delivery waiting to take its slot back once against the dispatch bound', async () => {+ internals().dispatchLimiter = new ConcurrencyLimiter(1, 1000);+ internals().dispatchMaxQueued = 2;+ for (const s of ['a', 'b', 'c', 'd']) {+ hooksBySession[`sess-${s}`] = [+ createMockWebhook({ id: `wh-${s}`, url: `https://${s}.example/hook`, retryCount: 2 }),+ ];+ }+ let wake: () => void = () => undefined;+ (sleep as unknown as jest.Mock).mockImplementationOnce(() => new Promise<void>(r => (wake = r)));+ let releaseB: () => void = () => undefined;+ let aCalls = 0;+ mockFetch.mockImplementation((url: string) =>+ url.startsWith('https://a.') && aCalls++ === 0+ ? Promise.reject(new Error('receiver down'))+ : url.startsWith('https://b.')+ ? new Promise(resolve => (releaseB = () => resolve({ ok: true, status: 200 })))+ : Promise.resolve({ ok: true, status: 200 }),+ );++ // wh-a sleeps without a slot, wh-b takes it, then wh-a wakes and parks to take it back.+ const a = service.dispatch('sess-a', 'message.received', {});+ await settle(() => (sleep as unknown as jest.Mock).mock.calls.length > 0);+ const b = service.dispatch('sess-b', 'message.received', {});+ await settle(() => mockFetch.mock.calls.length > 1);+ wake();+ await settle(() => internals().dispatchLimiter.queuedCount > 0);+ // Two deliveries are held against a bound of three: wh-c fits, wh-d does not.+ const c = service.dispatch('sess-c', 'message.received', {});+ await settle(() => internals().dispatchLimiter.queuedCount > 1 || insertedFailures.length > 0);+ const d = service.dispatch('sess-d', 'message.received', {});+ await settle(() => insertedFailures.length > 0);++ expect(insertedFailures).toEqual([+ expect.objectContaining({ webhookId: 'wh-d', attempts: 0, lastError: 'ConcurrencyLimiter queue full' }),+ ]);+ await d;+ releaseB();+ await Promise.all([a, b, c]);+ expect(urlsCalled()).toContain('https://c.example/hook');+ expect((outboxService.close.mock.calls as string[][]).map(call => call[0]).sort()).toEqual([+ 'wh-a',+ 'wh-b',+ 'wh-c',+ ]);+ });++ it('records a delivery parked behind the session gate as unsent at shutdown', async () => {+ internals().dispatchLimiter = new ConcurrencyLimiter(2, 1000);+ internals().degradedSessionConcurrency = 1;+ internals().failingWebhooks.add('wh-a1').add('wh-a2');+ (configService.get as jest.Mock).mockImplementation(<T>(key: string, def?: T): T | boolean | number =>+ key === 'webhook.shutdownDrainMs' ? 100 : key === 'queue.enabled' ? false : (def as T),+ );+ hooksBySession['sess-a'] = [+ createMockWebhook({ id: 'wh-a1', url: 'https://a1.example/hook', retryCount: 1 }),+ createMockWebhook({ id: 'wh-a2', url: 'https://a2.example/hook', retryCount: 1 }),+ ];+ const hanging: Array<() => void> = [];+ mockFetch.mockImplementation(+ () => new Promise(resolve => hanging.push(() => resolve({ ok: true, status: 200 }))),+ );++ const a = service.dispatch('sess-a', 'message.received', {});+ await settle(() => mockFetch.mock.calls.length > 0);+ await service.onModuleDestroy();++ expect(insertedFailures).toEqual([+ expect.objectContaining({ webhookId: 'wh-a2', attempts: 0, lastError: 'ConcurrencyLimiter closed' }),+ ]);+ hanging.forEach(release => release());+ await a;+ });+ });+describe('delivery-failure dead-letter', () => {it('records a durable failure when a direct delivery exhausts its retries', async () => {const webhook = createMockWebhook({ events: ['message.received'], retryCount: 1 });diff --git a/src/modules/webhook/webhook-delivery.service.ts b/src/modules/webhook/webhook-delivery.service.tsindex 4cbfc88c6..c79c4131f 100644--- a/src/modules/webhook/webhook-delivery.service.ts+++ b/src/modules/webhook/webhook-delivery.service.ts@@ -65,10 +65,13 @@ const DEFAULT_WEBHOOK_SHUTDOWN_DRAIN_MS = 5000;/** Per-event-occurrence context threaded through the dispatch pipeline stages (was closure state). *//**- * The result of one delivery attempt. Reported, not thrown: every failure below is already handled- * in place, so nothing reaches a caller as an exception and a try/catch cannot tell a delivered- * event from a dead-lettered one. 'cancelled' is a plugin suppressing the dispatch on purpose: it- * is terminal like 'delivered' and must never be replayed, but nothing left the process.+ * The result of one delivery attempt. Reported, not thrown: every delivery failure below is already+ * handled in place, so a try/catch cannot tell a delivered event from a dead-lettered one. The one+ * exception that escapes is 'ConcurrencyLimiter closed' from a retry backoff woken after shutdown;+ * only runLimited can see it, and it records the shutdown and keeps the outbox row pending.+ * 'cancelled' is terminal like 'delivered' and must never be replayed: either a plugin suppressed the+ * dispatch (nothing left the process), or a direct retry found the webhook removed, disabled or+ * unsubscribed (an earlier attempt may already have been POSTed).*/export type WebhookDeliveryOutcome = 'delivered' | 'enqueued' | 'cancelled' | 'failed';@@ -76,8 +79,14 @@ interface DispatchEventContext {sessionId: string;event: string;baseData: Record<string, unknown>;+ /** Gives the dispatch slot up while `fn` runs (a retry backoff). Absent outside the limiter. */+ yieldSlot?: <R>(fn: () => Promise<R>) => Promise<R>;}+/** The limiter refused or dropped the task because shutdown closed it. */+const isLimiterClosed = (error: unknown): boolean =>+ error instanceof Error && error.message === 'ConcurrencyLimiter closed';+/*** The webhook delivery engine: given an event occurrence, fan it out to the session's matching* webhooks — bounded by the dispatch limiter — through the BullMQ queue when enabled (with a@@ -92,10 +101,23 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {private readonly logger = createLogger('WebhookDelivery');private readonly queueEnabled: boolean;private readonly dispatchLimiter: ConcurrencyLimiter;+ private readonly dispatchMaxQueued: number;+ /**+ * Webhooks whose last delivery attempt on this node failed, until one succeeds. In direct mode a+ * delivery to one of them first passes its session's gate in `degradedSessionGates`, which admits+ * at most `degradedSessionConcurrency` at a time, so one session's failing receivers cannot keep+ * filling the dispatch pool while healthy receivers keep all of it. The gate is consulted once, at+ * admission: a delivery admitted before its webhook's first failure runs its retries ungated.+ */+ private readonly failingWebhooks = new Set<string>();+ private readonly degradedSessionGates = new Map<string, ConcurrencyLimiter>();+ private readonly degradedSessionConcurrency: number;/**- * Context of every delivery currently holding a dispatch-limiter slot (queued-path enqueue or a- * direct delivery with its retry loop). Used at shutdown to log, per delivery, what the bounded- * drain had to abandon — those deliveries were neither completed nor safely recorded.+ * Context of every delivery admitted to the dispatch limiter and not yet settled (queued-path+ * enqueue, or a direct delivery with its retry loop, including a backoff spent without a slot).+ * Counted against the dispatch budget in runLimited.+ * Used at shutdown to log, per delivery, what the bounded drain had to abandon: those deliveries+ * were neither completed nor safely recorded.*/private readonly inFlightDeliveries = new Map<string,@@ -129,10 +151,13 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {// Bound fan-out: cap how many matching webhooks are delivered CONCURRENTLY for one event. Without// it, an event matching N webhooks opens N outbound sockets at once. Default 16// (WEBHOOK_DISPATCH_CONCURRENCY).- this.dispatchLimiter = new ConcurrencyLimiter(- this.configService.get<number>('webhook.dispatchConcurrency', 16),- this.configService.get<number>('webhook.dispatchMaxQueued', 1000),- );+ const dispatchConcurrency = this.configService.get<number>('webhook.dispatchConcurrency', 16);+ this.dispatchMaxQueued = this.configService.get<number>('webhook.dispatchMaxQueued', 1000);+ this.dispatchLimiter = new ConcurrencyLimiter(dispatchConcurrency, this.dispatchMaxQueued);+ // WEBHOOK_DEGRADED_SESSION_CONCURRENCY, else a quarter of the pool.+ this.degradedSessionConcurrency =+ this.configService.get<number | undefined>('webhook.degradedSessionConcurrency') ??+ Math.max(1, Math.floor(dispatchConcurrency / 4));}onModuleInit(): void {@@ -160,6 +185,9 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {* skips the close: a parked dispatch's whole job is webhookQueue.add() — durable in Redis the* moment it resolves — so rejecting it would dead-letter work Redis could have kept. A parked* waiter holds an activeCount slot via handoff, so the drain loop below covers it either way.+ * A delivery waiting out a retry backoff holds no slot; if it wakes within the drain window, taking+ * one back fails on the closed limiter and it is recorded the same way. One still asleep when the+ * window ends is only logged below. Either way its outbox row stays pending for the next start.* In-flight deliveries (a direct delivery can outlive WEBHOOK_TIMEOUT via its backoff sleeps) get* up to WEBHOOK_SHUTDOWN_DRAIN_MS to finish; anything still running after that is about to be* dropped by process exit, so it is logged per delivery — a dead-letter row would be wrong there,@@ -169,13 +197,18 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {async onModuleDestroy(): Promise<void> {if (!this.queueEnabled) {this.dispatchLimiter.close();+ for (const gate of this.degradedSessionGates.values()) gate.close();}const drainMs = Math.max(0,this.configService.get<number>('webhook.shutdownDrainMs', DEFAULT_WEBHOOK_SHUTDOWN_DRAIN_MS),);const deadline = Date.now() + drainMs;- while (this.dispatchLimiter.activeCount > 0 || this.pendingBookkeeping.size > 0) {+ while (+ this.dispatchLimiter.activeCount > 0 ||+ this.inFlightDeliveries.size > 0 ||+ this.pendingBookkeeping.size > 0+ ) {const remaining = deadline - Date.now();if (remaining <= 0) break;await setTimeout(Math.min(50, remaining));@@ -429,7 +462,9 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {* What became of one delivery attempt, reported rather than thrown.** Every failure path here is already handled in place (a dead-letter row, a hook, a log), so none- * of them reach the caller as an exception. The reconciler has to tell a delivered event from a+ * of them reach the caller as an exception, except 'ConcurrencyLimiter closed' from a retry backoff+ * woken after shutdown, which runLimited handles (redeliver passes no yieldSlot and never sees it).+ * The reconciler has to tell a delivered event from a* dead-lettered one to know whether the outbox row may be retired, and a caught throw cannot tell* it: there is none. This mirrors the inbound twin, where `ingressEnqueue.enqueue` returns an* outcome and the caller retires the payload only when it is not 'failed'.@@ -530,7 +565,7 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {// same X-OpenWA-Idempotency-Key / X-OpenWA-Delivery-Id, so a conformant receiver dedupes.try {// Removed, disabled or unsubscribed before a retry: nothing to report, as on the queued path.- if (!(await this.deliverWebhook(webhook, finalPayload, body))) return 'cancelled';+ if (!(await this.deliverWebhook(webhook, finalPayload, body, ctx.yieldSlot))) return 'cancelled';await this.hookManager.execute('webhook:delivered',@@ -544,6 +579,9 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {{ sessionId, source: 'WebhookService' },);} catch (fallbackError) {+ // Shutdown closed the limiter during a retry backoff: not a delivery failure. runLimited+ // records it and keeps the outbox row pending for the next start.+ if (isLimiterClosed(fallbackError)) throw fallbackError;await this.hookManager.execute('webhook:error',{@@ -579,7 +617,7 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {const { sessionId, event } = ctx;try {// Removed, disabled or unsubscribed before a retry: nothing to report, as on the queued path.- if (!(await this.deliverWebhook(webhook, finalPayload, body))) return 'cancelled';+ if (!(await this.deliverWebhook(webhook, finalPayload, body, ctx.yieldSlot))) return 'cancelled';// Execute hook after successful deliveryawait this.hookManager.execute(@@ -595,6 +633,9 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {{ sessionId, source: 'WebhookService' },);} catch (error) {+ // Shutdown closed the limiter during a retry backoff: not a delivery failure. runLimited+ // records it and keeps the outbox row pending for the next start.+ if (isLimiterClosed(error)) throw error;// Execute hook on errorawait this.hookManager.execute('webhook:error',@@ -650,6 +691,57 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {return this.locallyPending.has(idempotencyKey);}+ /**+ * Direct mode: pass a delivery to a failing webhook through its session's gate before `run`, so+ * the deliveries that session admits after a failure hold at most `degradedSessionConcurrency`+ * dispatch slots however many of its receivers fail. Each gate parks at most a quarter of+ * WEBHOOK_DISPATCH_MAX_QUEUED, so a failing session sheds its own excess before it can spend the+ * park budget every session shares. Parking there also counts against WEBHOOK_DISPATCH_MAX_QUEUED+ * like parking in the dispatch limiter, and a delivery over it is shed the same way. Queued mode+ * holds a slot only for the enqueue, and the worker applies its own gate; the direct fallback for+ * a rejected enqueue is not gated.+ */+ private gateFailingReceiver(+ webhook: Webhook,+ sessionId: string,+ run: (gated: boolean) => Promise<void>,+ ): Promise<void> {+ if (this.queueEnabled || !this.failingWebhooks.has(webhook.id)) return run(false);+ let gate = this.degradedSessionGates.get(sessionId);+ if (!gate) {+ gate = new ConcurrencyLimiter(+ this.degradedSessionConcurrency,+ Math.max(1, Math.floor(this.dispatchMaxQueued / 4)),+ );+ this.degradedSessionGates.set(sessionId, gate);+ }+ if (gate.activeCount >= this.degradedSessionConcurrency && this.parkedDeliveries() >= this.dispatchMaxQueued) {+ return Promise.reject(new Error('ConcurrencyLimiter queue full'));+ }+ const current = gate;+ return current+ .run(() => run(true))+ .finally(() => {+ if (+ current.activeCount === 0 &&+ current.queuedCount === 0 &&+ this.degradedSessionGates.get(sessionId) === current+ ) {+ this.degradedSessionGates.delete(sessionId);+ }+ });+ }++ /**+ * Deliveries parked for a slot they were never given. One taking back the slot it gave up for a+ * retry backoff is left out: it is already counted in inFlightDeliveries.+ */+ private parkedDeliveries(): number {+ let parked = this.dispatchLimiter.queuedCount - this.dispatchLimiter.reacquiringCount;+ for (const gate of this.degradedSessionGates.values()) parked += gate.queuedCount;+ return parked;+ }+private async runLimited(webhook: Webhook,deliveryId: string,@@ -657,62 +749,105 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {ctx: DispatchEventContext,): Promise<void> {const { sessionId, event } = ctx;- await this.dispatchLimiter- .run(async () => {- this.inFlightDeliveries.set(deliveryId, {- webhookId: webhook.id,- sessionId,- event,+ // One budget for every admitted, unsettled delivery: running, parked in either limiter, or+ // asleep in a retry backoff that gave its slot up. The limiter's own maxQueued sees none of the+ // session gates or the sleepers, so on its own it no longer bounds what sits in heap.+ const overBudget =+ this.inFlightDeliveries.size + this.parkedDeliveries() >= this.dispatchLimiter.maxCount + this.dispatchMaxQueued;+ const admitted = overBudget+ ? Promise.reject(new Error('ConcurrencyLimiter queue full'))+ : this.gateFailingReceiver(webhook, sessionId, gated =>+ this.dispatchLimiter.run(async yieldSlot => {+ let target = webhook;+ if (gated) {+ // A delivery can sit behind the session gate far longer than the dispatch took, and the+ // gated webhooks are the ones an operator disables or re-points. Re-read the row before+ // the first attempt, as every retry does; healthy receivers never pay this read. If the+ // read fails, deliver with the dispatch-time row rather than drop the delivery silently:+ // every retry re-reads again, and a failed attempt is recorded as usual.+ let row: Webhook | null = webhook;+ try {+ row = await this.webhookRepository.findOne({ where: { id: webhook.id } });+ } catch (error) {+ this.logger.warn('Could not re-read parked webhook; delivering with the dispatch-time row', {+ webhookId: webhook.id,+ event,+ deliveryId,+ idempotencyKey,+ error: error instanceof Error ? error.message : String(error),+ action: 'webhook_stale_check_failed',+ });+ }+ if (!isDeliverableWebhook(row, event)) {+ this.logger.warn('Skipping parked webhook delivery: webhook removed, disabled or unsubscribed', {+ webhookId: webhook.id,+ event,+ deliveryId,+ idempotencyKey,+ action: 'webhook_skipped_stale',+ });+ this.failingWebhooks.delete(webhook.id);+ await this.outbox.close(webhook.id, idempotencyKey, 'dispatched');+ return;+ }+ target = row;+ }+ this.inFlightDeliveries.set(deliveryId, {+ webhookId: webhook.id,+ sessionId,+ event,+ idempotencyKey,+ url: target.url,+ });+ try {+ await this.deliverOne(target, deliveryId, idempotencyKey, { ...ctx, yieldSlot });+ // Reached a durable owner: handed to the queue, or completed inline. A failure inside+ // either owner dead-letters through the failure row, so this is never replayed.+ await this.outbox.close(webhook.id, idempotencyKey, 'dispatched');+ } finally {+ this.inFlightDeliveries.delete(deliveryId);+ }+ }),+ );+ await admitted.catch(async error => {+ if (error instanceof Error && error.message === 'ConcurrencyLimiter queue full') {+ // Shed before the task ran, so nothing was POSTed. The failure row reports the shed, but+ // the outbox row stays 'pending' on purpose: retiring it would drop the only copy of an+ // event the receiver provably never got. The sweep replays it once the backlog clears.+ await this.recordUndelivered(+ webhook,+ deliveryId,idempotencyKey,- url: webhook.url,- });+ error,+ 'webhook_dispatch_capacity_exceeded',+ ctx,+ );+ return;+ }+ if (error instanceof Error && error.message === 'ConcurrencyLimiter closed') {+ // Rejected by the shutdown drain before dispatching, or woken from a retry backoff after+ // it: record it like any other undelivered delivery, and track the write so+ // onModuleDestroy can await it (the limiter slot bookkeeping no longer covers this task).+ // Its outbox row stays 'pending': no POST was accepted, so the next start's sweep is what+ // finally delivers the event.+ const record = this.recordUndelivered(+ webhook,+ deliveryId,+ idempotencyKey,+ error,+ 'webhook_dispatch_shutdown',+ ctx,+ );+ this.pendingBookkeeping.add(record);try {- await this.deliverOne(webhook, deliveryId, idempotencyKey, ctx);- // Reached a durable owner: handed to the queue, or completed inline. A failure inside- // either owner dead-letters through the failure row, so this is never replayed.- await this.outbox.close(webhook.id, idempotencyKey, 'dispatched');+ await record;} finally {- this.inFlightDeliveries.delete(deliveryId);- }- })- .catch(async error => {- if (error instanceof Error && error.message === 'ConcurrencyLimiter queue full') {- // Shed before the task ran, so nothing was POSTed. The failure row reports the shed, but- // the outbox row stays 'pending' on purpose: retiring it would drop the only copy of an- // event the receiver provably never got. The sweep replays it once the backlog clears.- await this.recordUndelivered(- webhook,- deliveryId,- idempotencyKey,- error,- 'webhook_dispatch_capacity_exceeded',- ctx,- );- return;- }- if (error instanceof Error && error.message === 'ConcurrencyLimiter closed') {- // Rejected by the shutdown drain before dispatching — record it like any other- // undelivered delivery, and track the write so onModuleDestroy can await it (the- // limiter slot bookkeeping no longer covers this task). Its outbox row stays 'pending':- // the POST never happened, so the next start's sweep is what finally delivers the event.- const record = this.recordUndelivered(- webhook,- deliveryId,- idempotencyKey,- error,- 'webhook_dispatch_shutdown',- ctx,- );- this.pendingBookkeeping.add(record);- try {- await record;- } finally {- this.pendingBookkeeping.delete(record);- }- return;+ this.pendingBookkeeping.delete(record);}- throw error;- });+ return;+ }+ throw error;+ });}/**@@ -744,7 +879,12 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {* changed url, secret or header map applies from the next attempt. Retries back off exponentially* (retryDelay, then twice that, and so on), the schedule BullMQ applies to a queued job.*/- private async deliverWebhook(webhook: Webhook, payload: WebhookPayload, body: string): Promise<boolean> {+ private async deliverWebhook(+ webhook: Webhook,+ payload: WebhookPayload,+ body: string,+ yieldSlot: <R>(fn: () => Promise<R>) => Promise<R> = fn => fn(),+ ): Promise<boolean> {const delay = this.configService.get<number>('webhook.retryDelay', 5000);let current = webhook;for (let attempt = 1; ; attempt++) {@@ -760,6 +900,7 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {idempotencyKey: payload.idempotencyKey,action: 'webhook_skipped_stale',});+ this.failingWebhooks.delete(webhook.id);return false;}current = row;@@ -773,6 +914,7 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {attempt - 1,);await postWebhookPayload(current.url, body, headers, this.configService.get<number>('webhook.timeout', 10000));+ this.failingWebhooks.delete(webhook.id);// The receiver already answered 2xx — the delivery SUCCEEDED. A bookkeeping failure here (e.g.// the lastTriggeredAt update on a flaky DB) must not reach the catch below: it would retry an@@ -800,6 +942,7 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {});return true;} catch (error) {+ this.failingWebhooks.add(webhook.id);this.logger.error(`Webhook delivery failed for ${webhook.id}`, String(error), {webhookId: webhook.id,attempt,@@ -808,7 +951,10 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {});if (attempt < current.retryCount) {- await setTimeout(delay * 2 ** (attempt - 1));+ // Without the dispatch slot: a backoff sends nothing, and holding the slot through it let+ // a few failing receivers stall every other delivery. Taking it back throws once shutdown+ // closed the limiter, which skips the terminal record below: the event was not given up.+ await yieldSlot(() => setTimeout(delay * 2 ** (attempt - 1)));continue;}// All direct-path retries exhausted — persist a durable failure record before giving up, mirroring