Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(webhook): re-read the webhook before each direct retry

Yudhi Armyndharis858beb2a0626
diff --git a/docs/06-api-specification.md b/docs/06-api-specification.md
index e222a0ab2..f795b1fff 100644
--- a/docs/06-api-specification.md
+++ b/docs/06-api-specification.md
@@ -7141,7 +7141,7 @@ Recurring lifecycle events (and `message.reaction` / `message.edited`) carry the
### Retries with exponential backoff
-When the queue is enabled, a non-2xx response, timeout (`WEBHOOK_TIMEOUT`, default `10000` ms), or network error schedules a retry. The number of attempts comes from the webhook's `retryCount` (default `3`) and the delay grows **exponentially** from a base of `WEBHOOK_RETRY_DELAY` (default `5000` ms). Each retry reuses the same `idempotencyKey` and increments `X-OpenWA-Retry-Count`. If Redis/BullMQ rejects the initial enqueue, OpenWA logs a `webhook:error` hook event and falls back to direct delivery with the same inline retry budget. When the queue is disabled, delivery is direct with the same retry budget applied inline.
+When the queue is enabled, a non-2xx response, timeout (`WEBHOOK_TIMEOUT`, default `10000` ms), or network error schedules a retry. The number of attempts comes from the webhook's `retryCount` (default `3`) and the delay grows **exponentially** from a base of `WEBHOOK_RETRY_DELAY` (default `5000` ms). Each retry reuses the same `idempotencyKey` and increments `X-OpenWA-Retry-Count`. If Redis/BullMQ rejects the initial enqueue, OpenWA logs a `webhook:error` hook event and falls back to direct delivery with the same inline retry budget. When the queue is disabled, delivery is direct with the same retry budget and the same exponential backoff applied inline. Every retry, queued or direct, re-reads the webhook first: a webhook deleted, disabled or unsubscribed from the event since the dispatch gets no further attempt and no delivery-failure row, and a changed URL, secret or header map applies from the next attempt.
### SSRF guard on registration
diff --git a/src/modules/queue/processors/webhook.processor.ts b/src/modules/queue/processors/webhook.processor.ts
index e2ea06f5d..9505aad9d 100644
--- a/src/modules/queue/processors/webhook.processor.ts
+++ b/src/modules/queue/processors/webhook.processor.ts
@@ -14,7 +14,7 @@ import {
recordWebhookDeliveryFailure,
statusCodeFromError,
} from '../../webhook/utils/record-delivery-failure';
-import { buildDeliveryHeaders, postWebhookPayload } from '../../webhook/utils/deliver-once';
+import { buildDeliveryHeaders, isDeliverableWebhook, postWebhookPayload } from '../../webhook/utils/deliver-once';
import { HookManager } from '../../../core/hooks';
import { redactSsrfError } from '../../../common/security/ssrf-guard';
import { incrementWebhookDeliveryFailures } from '../../../common/metrics/webhook-delivery-metrics';
@@ -139,7 +139,7 @@ export class WebhookProcessor extends WorkerHost {
/** The webhook row as it is now, or null when it was deleted, disabled or no longer takes `event`. */
private async loadDeliverableWebhook(webhookId: string, event: string): Promise<Webhook | null> {
const row = await this.webhookRepository.findOne({ where: { id: webhookId } });
- return row && row.active && (row.events.includes(event) || row.events.includes('*')) ? row : null;
+ return isDeliverableWebhook(row, event) ? row : null;
}
/**
diff --git a/src/modules/webhook/utils/deliver-once.spec.ts b/src/modules/webhook/utils/deliver-once.spec.ts
index 8648129dd..2e45986d8 100644
--- a/src/modules/webhook/utils/deliver-once.spec.ts
+++ b/src/modules/webhook/utils/deliver-once.spec.ts
@@ -1,5 +1,5 @@
import { Headers } from 'undici';
-import { buildDeliveryHeaders, postWebhookPayload, sanitizeCustomHeaders } from './deliver-once';
+import { buildDeliveryHeaders, isDeliverableWebhook, postWebhookPayload, sanitizeCustomHeaders } from './deliver-once';
/**
* Direct coverage for the shared delivery core both paths (direct and queued processor) now route
@@ -95,3 +95,19 @@ describe('sanitizeCustomHeaders', () => {
).toEqual({ 'X-Keep': 'v' });
});
});
+
+describe('isDeliverableWebhook', () => {
+ const row = { active: true, events: ['message.received'] };
+
+ it('accepts an active row subscribed to the event or to every event', () => {
+ expect(isDeliverableWebhook(row, 'message.received')).toBe(true);
+ expect(isDeliverableWebhook({ active: true, events: ['*'] }, 'message.ack')).toBe(true);
+ });
+
+ it('refuses a missing, disabled or unsubscribed row', () => {
+ expect(isDeliverableWebhook(null, 'message.received')).toBe(false);
+ expect(isDeliverableWebhook(undefined, 'message.received')).toBe(false);
+ expect(isDeliverableWebhook({ ...row, active: false }, 'message.received')).toBe(false);
+ expect(isDeliverableWebhook(row, 'message.ack')).toBe(false);
+ });
+});
diff --git a/src/modules/webhook/utils/deliver-once.ts b/src/modules/webhook/utils/deliver-once.ts
index 7f7e6c14f..23ffdfe38 100644
--- a/src/modules/webhook/utils/deliver-once.ts
+++ b/src/modules/webhook/utils/deliver-once.ts
@@ -26,6 +26,19 @@ export function sanitizeCustomHeaders(custom: Record<string, string> | null | un
return safe;
}
+/**
+ * Whether a webhook row as read now may still receive `event`: it exists, is active and subscribes to
+ * the event or to '*'. Its filters are not re-applied here, since they need the event data. Every
+ * path that delivers after the dispatch moment (a queued job, a direct retry, an outbox replay) runs
+ * this against a fresh row, so a removed, disabled or unsubscribed webhook stops receiving the event.
+ */
+export function isDeliverableWebhook<T extends { active: boolean; events: string[] }>(
+ row: T | null | undefined,
+ event: string,
+): row is T {
+ return !!row && row.active && (row.events.includes(event) || row.events.includes('*'));
+}
+
/** HMAC-SHA256 over the exact pre-serialized body, prefixed for receiver-side verification. */
export function generateSignature(payload: string, secret: string): string {
const hmac = crypto.createHmac('sha256', secret);
diff --git a/src/modules/webhook/webhook-delivery.service.spec.ts b/src/modules/webhook/webhook-delivery.service.spec.ts
index a1ad31aec..eb746d113 100644
--- a/src/modules/webhook/webhook-delivery.service.spec.ts
+++ b/src/modules/webhook/webhook-delivery.service.spec.ts
@@ -10,6 +10,14 @@ jest.mock('undici', () => {
return { __esModule: true, ...actual, fetch: jest.fn() };
});
+// A pass-through spy, so a test can read the backoff the retry loop asked for without faking time
+// (jest's fake timers do not cover node:timers/promises).
+jest.mock('node:timers/promises', () => {
+ const actual = jest.requireActual<typeof import('node:timers/promises')>('node:timers/promises');
+ return { ...actual, setTimeout: jest.fn(actual.setTimeout) };
+});
+
+import { setTimeout as sleep } from 'node:timers/promises';
import { Test, TestingModule } from '@nestjs/testing';
import { getRepositoryToken } from '@nestjs/typeorm';
import { getQueueToken } from '@nestjs/bullmq';
@@ -73,6 +81,12 @@ describe('WebhookDeliveryService', () => {
beforeEach(async () => {
repository = {
find: jest.fn(),
+ // A direct retry re-reads its webhook. By default it reads back the row dispatch loaded, so a
+ // test changes the webhook between attempts by overriding this.
+ findOne: jest.fn().mockImplementation(async (opts: { where: { id: string } }) => {
+ const rows = (await (repository.find as jest.Mock)()) as Webhook[] | undefined;
+ return rows?.find(w => w.id === opts.where.id) ?? null;
+ }),
update: jest.fn().mockResolvedValue({ affected: 1 }),
};
@@ -1714,6 +1728,25 @@ describe('WebhookDeliveryService', () => {
);
});
+ it('reports cancelled when the queue fallback finds the webhook gone before a retry', async () => {
+ const queueService = await buildQueueService(<T>(key: string, def?: T): T | boolean | number => {
+ if (key === 'queue.enabled') return true;
+ if (key === 'webhook.retryDelay') return 10;
+ return def as T;
+ });
+ const webhook = createMockWebhook({ retryCount: 2 });
+ (repository.findOne as jest.Mock).mockResolvedValue(null);
+ webhookQueue.add.mockRejectedValueOnce(new Error('redis down'));
+ const mockFetch = undiciFetch as jest.Mock;
+ (hookManager.execute as jest.Mock).mockResolvedValue({ continue: true, data: {} });
+ mockFetch.mockRejectedValue(new Error('receiver down'));
+
+ await expect(queueService.redeliver(webhook, 'sess-1', 'message.received', 'k', {})).resolves.toBe('cancelled');
+
+ expect(mockFetch).toHaveBeenCalledTimes(1);
+ expect(failureRepository.insert).not.toHaveBeenCalled();
+ });
+
it('falls back to direct delivery when queue add fails', async () => {
const queueService = await buildQueueService(<T>(key: string, def?: T): T | boolean | number => {
if (key === 'queue.enabled') return true;
@@ -1755,6 +1788,92 @@ describe('WebhookDeliveryService', () => {
});
});
+ describe('direct retries', () => {
+ const mockFetch = undiciFetch as jest.Mock;
+ type SentInit = { headers: Record<string, string>; body: string };
+
+ beforeEach(() => {
+ (hookManager.execute as jest.Mock).mockResolvedValue({ continue: true, data: {} });
+ mockFetch.mockReset();
+ });
+
+ afterEach(() => mockFetch.mockReset());
+
+ // A queued job re-reads its webhook before every attempt; the direct loop kept POSTing the
+ // snapshot taken at dispatch, then filed a dead-letter row for a webhook the operator removed.
+ it.each([
+ ['deleted', null],
+ ['disabled', { active: false }],
+ ['unsubscribed from the event', { events: ['message.ack'] }],
+ ])('stops retrying, files nothing and reports cancelled when the webhook was %s', async (_label, change) => {
+ const webhook = createMockWebhook({ retryCount: 3 });
+ (repository.findOne as jest.Mock).mockResolvedValue(change && { ...webhook, ...change });
+ mockFetch.mockRejectedValue(new Error('receiver down'));
+ const failuresBefore = getWebhookDeliveryFailuresTotal();
+
+ await expect(service.redeliver(webhook, 'sess-1', 'message.received', 'k', {})).resolves.toBe('cancelled');
+
+ expect(mockFetch).toHaveBeenCalledTimes(1);
+ expect(failureRepository.insert).not.toHaveBeenCalled();
+ expect(getWebhookDeliveryFailuresTotal()).toBe(failuresBefore);
+ const hooks = (hookManager.execute as jest.Mock).mock.calls.map(([name]) => name as string);
+ expect(hooks).not.toContain('webhook:delivered');
+ expect(hooks).not.toContain('webhook:after');
+ });
+
+ it('sends a retry to the current url, signed with the current secret', async () => {
+ const webhook = createMockWebhook({ retryCount: 2, secret: 'old-secret-0123456789' });
+ const moved = { ...webhook, url: 'https://moved.example/hook', secret: 'new-secret-0123456789' };
+ (repository.findOne as jest.Mock).mockResolvedValue(moved);
+ mockFetch.mockRejectedValueOnce(new Error('receiver down')).mockResolvedValueOnce({ ok: true, status: 200 });
+
+ await expect(service.redeliver(webhook, 'sess-1', 'message.received', 'k', {})).resolves.toBe('delivered');
+
+ const calls = mockFetch.mock.calls as Array<[string, SentInit]>;
+ expect(calls.map(([url]) => url)).toEqual(['https://example.com/webhook', 'https://moved.example/hook']);
+ const [, retry] = calls[1];
+ const expected = `sha256=${crypto.createHmac('sha256', 'new-secret-0123456789').update(retry.body).digest('hex')}`;
+ expect(retry.headers['X-OpenWA-Signature']).toBe(expected);
+ expect(retry.headers['X-OpenWA-Retry-Count']).toBe('1');
+ });
+
+ it('drops the signature from a retry once the secret is cleared', async () => {
+ const webhook = createMockWebhook({ retryCount: 2, secret: 'old-secret-0123456789' });
+ (repository.findOne as jest.Mock).mockResolvedValue({ ...webhook, secret: null });
+ mockFetch.mockRejectedValueOnce(new Error('receiver down')).mockResolvedValueOnce({ ok: true, status: 200 });
+
+ await service.redeliver(webhook, 'sess-1', 'message.received', 'k', {});
+
+ const calls = mockFetch.mock.calls as Array<[string, SentInit]>;
+ expect(calls[0][1].headers).toHaveProperty('X-OpenWA-Signature');
+ expect(calls[1][1].headers).not.toHaveProperty('X-OpenWA-Signature');
+ });
+
+ it('counts a failed re-read as a failed attempt', async () => {
+ const webhook = createMockWebhook({ retryCount: 2 });
+ (repository.findOne as jest.Mock).mockRejectedValue(new Error('db down'));
+ mockFetch.mockRejectedValue(new Error('receiver down'));
+
+ await expect(service.redeliver(webhook, 'sess-1', 'message.received', 'k', {})).resolves.toBe('failed');
+
+ expect(mockFetch).toHaveBeenCalledTimes(1);
+ expect(insertedFailures).toEqual([expect.objectContaining({ attempts: 2, lastError: 'db down' })]);
+ });
+
+ it('backs off exponentially between attempts, as the queue does', async () => {
+ const webhook = createMockWebhook({ retryCount: 4 });
+ (repository.findOne as jest.Mock).mockResolvedValue(webhook);
+ mockFetch.mockRejectedValue(new Error('receiver down'));
+ (sleep as unknown as jest.Mock).mockClear();
+
+ await expect(service.redeliver(webhook, 'sess-1', 'message.received', 'k', {})).resolves.toBe('failed');
+
+ // retryDelay is 100 here: 100, then 200, then 400.
+ expect((sleep as unknown as jest.Mock).mock.calls.map(([ms]) => ms as number)).toEqual([100, 200, 400]);
+ expect(mockFetch).toHaveBeenCalledTimes(4);
+ });
+ });
+
describe('delivery-failure dead-letter', () => {
it('records a durable failure when a direct delivery exhausts its retries', async () => {
const webhook = createMockWebhook({ events: ['message.received'], retryCount: 1 });
diff --git a/src/modules/webhook/webhook-delivery.service.ts b/src/modules/webhook/webhook-delivery.service.ts
index 27ba4f782..4cbfc88c6 100644
--- a/src/modules/webhook/webhook-delivery.service.ts
+++ b/src/modules/webhook/webhook-delivery.service.ts
@@ -9,7 +9,12 @@ import { Webhook } from './entities/webhook.entity';
import { WebhookOutboxService } from './webhook-outbox.service';
import { WebhookDeliveryFailure } from './entities/webhook-delivery-failure.entity';
import { clearDeliveryFailureRows, recordWebhookDeliveryFailure } from './utils/record-delivery-failure';
-import { buildDeliveryHeaders, postWebhookPayload, recordTerminalFailure } from './utils/deliver-once';
+import {
+ buildDeliveryHeaders,
+ isDeliverableWebhook,
+ postWebhookPayload,
+ recordTerminalFailure,
+} from './utils/deliver-once';
import { createLogger } from '../../common/services/logger.service';
import { DEFAULT_WEBHOOK_MEDIA_INLINE_MAX_BYTES, shedInlineMedia } from '../../common/utils/inline-media';
import { incrementWebhookDeliveryFailures } from '../../common/metrics/webhook-delivery-metrics';
@@ -300,8 +305,8 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
}
/**
- * Build one webhook delivery: payload + webhook:before hooks + identity re-assertion + size gate +
- * headers. Returns null when the delivery must not proceed — either cancelled by a plugin (a debug
+ * Build one webhook delivery: payload + webhook:before hooks + identity re-assertion + size gate.
+ * Returns null when the delivery must not proceed: either cancelled by a plugin (a debug
* log, not a failure) or after a failure already recorded via recordUndelivered.
*/
private async preflightDelivery(
@@ -309,7 +314,7 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
deliveryId: string,
idempotencyKey: string,
ctx: DispatchEventContext,
- ): Promise<{ finalPayload: WebhookPayload; body: string; headers: Record<string, string> } | 'cancelled' | null> {
+ ): Promise<{ finalPayload: WebhookPayload; body: string } | 'cancelled' | null> {
const { sessionId, event, baseData } = ctx;
try {
const payload: WebhookPayload = {
@@ -406,8 +411,7 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
return null;
}
- const headers = buildDeliveryHeaders(webhook, event, idempotencyKey, deliveryId, body);
- return { finalPayload, body, headers };
+ return { finalPayload, body };
} catch (error) {
await this.recordUndelivered(
webhook,
@@ -447,23 +451,22 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
// The remaining bail-outs record their own undelivered row before returning null.
return 'failed';
}
- const { finalPayload, body, headers } = preflight;
+ const { finalPayload, body } = preflight;
// Use queue if available, otherwise fallback to direct delivery
if (this.queueEnabled && this.webhookQueue) {
// A replay's attempts-0 row is not cleared here. It stays until the delivery resolves: a
// successful POST (the processor's, or the fallback's when the add fails) clears it, and a
// terminal failure replaces it right after filing its own row. A restart or a lost job at
// any point before that still leaves the event on record.
- return this.enqueueWithFallback(webhook, finalPayload, body, headers, deliveryId, idempotencyKey, ctx);
+ return this.enqueueWithFallback(webhook, finalPayload, body, deliveryId, idempotencyKey, ctx);
}
- return this.deliverDirect(webhook, finalPayload, body, headers, deliveryId, ctx);
+ return this.deliverDirect(webhook, finalPayload, body, deliveryId, ctx);
}
private async enqueueWithFallback(
webhook: Webhook,
finalPayload: WebhookPayload,
body: string,
- headers: Record<string, string>,
deliveryId: string,
idempotencyKey: string,
ctx: DispatchEventContext,
@@ -526,7 +529,8 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
// Redis before rejecting, the queued job AND this fallback may both POST. Both paths carry the
// same X-OpenWA-Idempotency-Key / X-OpenWA-Delivery-Id, so a conformant receiver dedupes.
try {
- await this.deliverWebhook(webhook, finalPayload, headers, body);
+ // Removed, disabled or unsubscribed before a retry: nothing to report, as on the queued path.
+ if (!(await this.deliverWebhook(webhook, finalPayload, body))) return 'cancelled';
await this.hookManager.execute(
'webhook:delivered',
@@ -569,13 +573,13 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
webhook: Webhook,
finalPayload: WebhookPayload,
body: string,
- headers: Record<string, string>,
deliveryId: string,
ctx: DispatchEventContext,
): Promise<WebhookDeliveryOutcome> {
const { sessionId, event } = ctx;
try {
- await this.deliverWebhook(webhook, finalPayload, headers, body);
+ // Removed, disabled or unsubscribed before a retry: nothing to report, as on the queued path.
+ if (!(await this.deliverWebhook(webhook, finalPayload, body))) return 'cancelled';
// Execute hook after successful delivery
await this.hookManager.execute(
@@ -732,77 +736,100 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
/**
* Direct delivery with in-process retries, up to `webhook.retryCount` attempts: the path every
* delivery takes when the queue is disabled (the default), and the fallback when an enqueue fails.
- * `body` is the pre-serialized payload from preflight — the exact bytes the size gate checked and
- * (when a secret is set) the signature covers — so it is never re-serialized here.
+ * `body` is the pre-serialized payload from preflight, the exact bytes the size gate checked, so it
+ * is never re-serialized here.
+ *
+ * Like a queued job, every retry re-reads the webhook row: a webhook removed, disabled or
+ * unsubscribed since the dispatch gets nothing more and no failure row (resolves false), and a
+ * changed url, secret or header map applies from the next attempt. Retries back off exponentially
+ * (retryDelay, then twice that, and so on), the schedule BullMQ applies to a queued job.
*/
- private async deliverWebhook(
- webhook: Webhook,
- payload: WebhookPayload,
- headers: Record<string, string>,
- body: string,
- attempt = 1,
- ): Promise<void> {
- // Update retry count header
- headers['X-OpenWA-Retry-Count'] = String(attempt - 1);
-
- try {
- await postWebhookPayload(webhook.url, body, headers, this.configService.get<number>('webhook.timeout', 10000));
-
- // The receiver already answered 2xx — the delivery SUCCEEDED. A bookkeeping failure here (e.g.
- // the lastTriggeredAt update on a flaky DB) must not reach the catch below: it would retry an
- // already-delivered webhook (duplicate POST) and, on the last attempt, file a false dead-letter
- // row. Log it and keep the success outcome.
+ private async deliverWebhook(webhook: Webhook, payload: WebhookPayload, body: string): Promise<boolean> {
+ const delay = this.configService.get<number>('webhook.retryDelay', 5000);
+ let current = webhook;
+ for (let attempt = 1; ; attempt++) {
try {
- await this.webhookRepository.update(webhook.id, {
- lastTriggeredAt: new Date(),
- });
- } catch (bookkeepingError) {
- this.logger.error(
- `Webhook delivered to ${webhook.id} but lastTriggeredAt update failed`,
- bookkeepingError instanceof Error ? bookkeepingError.message : String(bookkeepingError),
- { webhookId: webhook.id, deliveryId: payload.deliveryId, action: 'webhook_bookkeeping_failed' },
+ if (attempt > 1) {
+ // Inside the try: a read error counts as a failed attempt, as it does for a queued job.
+ const row = await this.webhookRepository.findOne({ where: { id: webhook.id } });
+ if (!isDeliverableWebhook(row, payload.event)) {
+ this.logger.warn('Skipping webhook retry: webhook removed, disabled or unsubscribed', {
+ webhookId: webhook.id,
+ event: payload.event,
+ deliveryId: payload.deliveryId,
+ idempotencyKey: payload.idempotencyKey,
+ action: 'webhook_skipped_stale',
+ });
+ return false;
+ }
+ current = row;
+ }
+ const headers = buildDeliveryHeaders(
+ current,
+ payload.event,
+ payload.idempotencyKey,
+ payload.deliveryId,
+ body,
+ attempt - 1,
);
- }
- // A delivered event must not stay listed as lost: a replay of a shed, refused or failed
- // dispatch, or a re-emitted event, reuses the key an earlier failure row was filed under.
- await clearDeliveryFailureRows(this.failureRepository, this.logger, webhook.id, payload.idempotencyKey);
+ await postWebhookPayload(current.url, body, headers, this.configService.get<number>('webhook.timeout', 10000));
- this.logger.debug(`Webhook delivered to ${webhook.id}`, {
- webhookId: webhook.id,
- deliveryId: payload.deliveryId,
- action: 'webhook_delivered',
- });
- } catch (error) {
- this.logger.error(`Webhook delivery failed for ${webhook.id}`, String(error), {
- webhookId: webhook.id,
- attempt,
- deliveryId: payload.deliveryId,
- action: 'webhook_delivery_failed',
- });
+ // The receiver already answered 2xx — the delivery SUCCEEDED. A bookkeeping failure here (e.g.
+ // the lastTriggeredAt update on a flaky DB) must not reach the catch below: it would retry an
+ // already-delivered webhook (duplicate POST) and, on the last attempt, file a false dead-letter
+ // row. Log it and keep the success outcome.
+ try {
+ await this.webhookRepository.update(webhook.id, {
+ lastTriggeredAt: new Date(),
+ });
+ } catch (bookkeepingError) {
+ this.logger.error(
+ `Webhook delivered to ${webhook.id} but lastTriggeredAt update failed`,
+ bookkeepingError instanceof Error ? bookkeepingError.message : String(bookkeepingError),
+ { webhookId: webhook.id, deliveryId: payload.deliveryId, action: 'webhook_bookkeeping_failed' },
+ );
+ }
+ // A delivered event must not stay listed as lost: a replay of a shed, refused or failed
+ // dispatch, or a re-emitted event, reuses the key an earlier failure row was filed under.
+ await clearDeliveryFailureRows(this.failureRepository, this.logger, webhook.id, payload.idempotencyKey);
- if (attempt < webhook.retryCount) {
- const delay = this.configService.get<number>('webhook.retryDelay', 5000);
- await setTimeout(delay * attempt);
- return this.deliverWebhook(webhook, payload, headers, body, attempt + 1);
- }
- // All direct-path retries exhausted — persist a durable failure record before giving up, mirroring
- // the queued processor's final-attempt path so the queue-disabled path isn't a blind spot.
- // The recorder writes this row first and only then removes an attempts-0 row of the same
- // delivery (a shed or refused dispatch), so a restart at any point keeps a record.
- const recorded = await recordTerminalFailure(this.failureRepository, this.logger, {
- webhookId: webhook.id,
- sessionId: payload.sessionId,
- event: payload.event,
- url: webhook.url,
- idempotencyKey: payload.idempotencyKey,
- deliveryId: payload.deliveryId,
- attempts: attempt,
- error,
- });
- if (recorded) {
- incrementWebhookDeliveryFailures();
+ this.logger.debug(`Webhook delivered to ${webhook.id}`, {
+ webhookId: webhook.id,
+ deliveryId: payload.deliveryId,
+ action: 'webhook_delivered',
+ });
+ return true;
+ } catch (error) {
+ this.logger.error(`Webhook delivery failed for ${webhook.id}`, String(error), {
+ webhookId: webhook.id,
+ attempt,
+ deliveryId: payload.deliveryId,
+ action: 'webhook_delivery_failed',
+ });
+
+ if (attempt < current.retryCount) {
+ await setTimeout(delay * 2 ** (attempt - 1));
+ continue;
+ }
+ // All direct-path retries exhausted — persist a durable failure record before giving up, mirroring
+ // the queued processor's final-attempt path so the queue-disabled path isn't a blind spot.
+ // The recorder writes this row first and only then removes an attempts-0 row of the same
+ // delivery (a shed or refused dispatch), so a restart at any point keeps a record.
+ const recorded = await recordTerminalFailure(this.failureRepository, this.logger, {
+ webhookId: webhook.id,
+ sessionId: payload.sessionId,
+ event: payload.event,
+ url: current.url,
+ idempotencyKey: payload.idempotencyKey,
+ deliveryId: payload.deliveryId,
+ attempts: attempt,
+ error,
+ });
+ if (recorded) {
+ incrementWebhookDeliveryFailures();
+ }
+ throw error;
}
- throw error;
}
}
}
diff --git a/src/modules/webhook/webhook-reconciler.service.ts b/src/modules/webhook/webhook-reconciler.service.ts
index a6e324ec1..22a415b2e 100644
--- a/src/modules/webhook/webhook-reconciler.service.ts
+++ b/src/modules/webhook/webhook-reconciler.service.ts
@@ -4,6 +4,7 @@ import { Repository } from 'typeorm';
import { Webhook } from './entities/webhook.entity';
import { WebhookOutboxService } from './webhook-outbox.service';
import { WebhookDeliveryService } from './webhook-delivery.service';
+import { isDeliverableWebhook } from './utils/deliver-once';
import { createLogger } from '../../common/services/logger.service';
import { resolveNonNegativeIntEnv } from '../../config/configuration';
@@ -108,7 +109,7 @@ export class WebhookReconcilerService implements OnModuleInit, OnModuleDestroy {
continue;
}
const webhook = await this.webhooks.findOne({ where: { id: row.webhookId } });
- if (!webhook || !webhook.active || !(webhook.events.includes(row.event) || webhook.events.includes('*'))) {
+ if (!isDeliverableWebhook(webhook, row.event)) {
// The subscription is gone, switched off or no longer lists this event; replaying it would
// deliver an event the operator has already unsubscribed from. Same test as the queue
// processor applies before every attempt.
@@ -140,8 +141,9 @@ export class WebhookReconcilerService implements OnModuleInit, OnModuleDestroy {
stats.failed++;
continue;
}
- // 'delivered', 'enqueued' and 'cancelled' all retire the row: the delivery either reached a
- // durable owner or a plugin dropped it on purpose. Only 'failed' is worth another sweep.
+ // 'delivered', 'enqueued' and 'cancelled' all retire the row: the delivery reached a durable
+ // owner, a plugin dropped it on purpose, or a retry found the webhook removed, disabled or
+ // unsubscribed. Only 'failed' is worth another sweep.
await this.outbox.close(row.webhookId, row.idempotencyKey, 'dispatched');
stats.replayed++;
} catch (error) {