Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(webhooks): keep recovery deliveries current

Yudhi Armyndharis94bdf4402d83
diff --git a/src/modules/queue/processors/webhook.processor.spec.ts b/src/modules/queue/processors/webhook.processor.spec.ts
index ba4e99c84..d8951e701 100644
--- a/src/modules/queue/processors/webhook.processor.spec.ts
+++ b/src/modules/queue/processors/webhook.processor.spec.ts
@@ -61,7 +61,13 @@ describe('WebhookProcessor', () => {
repo = {
update: jest.fn().mockResolvedValue({ affected: 1 }),
// The live row matches makeJob()'s snapshot unless a test says otherwise.
- findOne: jest.fn().mockResolvedValue({ id: 'wh-1', active: true, url: 'https://8.8.8.8/hook', events: ['*'] }),
+ findOne: jest.fn().mockResolvedValue({
+ id: 'wh-1',
+ active: true,
+ sessionId: 'sess-1',
+ url: 'https://8.8.8.8/hook',
+ events: ['*'],
+ }),
};
// Stateful like the real table: the recorder counts existing rows for the delivery before it
// inserts, so a constant would leave that guard unexercised here and let a duplicated row pass.
@@ -173,7 +179,13 @@ describe('WebhookProcessor', () => {
it('persists a durable delivery-failure record on the final attempt (with parsed HTTP status)', async () => {
mockFetch.mockResolvedValue({ ok: false, status: 503, statusText: 'Service Unavailable' });
- repo.findOne.mockResolvedValue({ id: 'wh-x', active: true, url: 'https://8.8.8.8/h', events: ['*'] });
+ repo.findOne.mockResolvedValue({
+ id: 'wh-x',
+ active: true,
+ sessionId: 'sess-1',
+ url: 'https://8.8.8.8/h',
+ events: ['*'],
+ });
await expect(
processor.process(makeJob({ maxRetries: 3, webhookId: 'wh-x', url: 'https://8.8.8.8/h' }, 2)),
@@ -300,7 +312,13 @@ describe('WebhookProcessor', () => {
// message — the resolved internal IP is a recon oracle. The server-side logger.error keeps full detail.
it('redacts the resolved internal IP from the webhook:error payload and DLQ row on an SSRF block', async () => {
process.env.WEBHOOK_SSRF_PROTECT = 'true';
- repo.findOne.mockResolvedValue({ id: 'wh-1', active: true, url: 'https://169.254.169.254/h', events: ['*'] });
+ repo.findOne.mockResolvedValue({
+ id: 'wh-1',
+ active: true,
+ sessionId: 'sess-1',
+ url: 'https://169.254.169.254/h',
+ events: ['*'],
+ });
// final attempt (attemptsMade=0, maxRetries=1 → 1 >= 1) so the hook + DLQ fire
await expect(processor.process(makeJob({ url: 'https://169.254.169.254/h', maxRetries: 1 }, 0))).rejects.toThrow();
@@ -362,12 +380,13 @@ describe('WebhookProcessor', () => {
// The job is a snapshot from enqueue time; the operator may have changed the webhook since.
describe('stale snapshot', () => {
- const live = { id: 'wh-1', active: true, url: 'https://8.8.8.8/hook', events: ['*'] };
+ const live = { id: 'wh-1', active: true, sessionId: 'sess-1', url: 'https://8.8.8.8/hook', events: ['*'] };
it.each([
['deleted', null],
['disabled', { ...live, active: false }],
['unsubscribed', { ...live, events: ['message.sent'] }],
+ ['reassigned', { ...live, sessionId: 'sess-2' }],
])('does not POST, retry or dead-letter a job for a %s webhook', async (_label, row) => {
repo.findOne.mockResolvedValue(row);
mockFetch.mockResolvedValue({ ok: true, status: 200 });
@@ -433,7 +452,13 @@ describe('WebhookProcessor', () => {
it('records a dead-letter row, metric, and webhook:error for a job failed by a double stall', async () => {
const failuresBefore = getWebhookDeliveryFailuresTotal();
// Re-pointed since enqueue: the row names the URL a retry would have used, not the snapshot's.
- repo.findOne.mockResolvedValue({ id: 'wh-stall', active: true, url: 'https://8.8.8.8/s', events: ['*'] });
+ repo.findOne.mockResolvedValue({
+ id: 'wh-stall',
+ active: true,
+ sessionId: 'sess-1',
+ url: 'https://8.8.8.8/s',
+ events: ['*'],
+ });
await processor.onWorkerFailed(
makeJob({ webhookId: 'wh-stall', url: 'https://8.8.8.8/old' }, 1),
@@ -462,6 +487,7 @@ describe('WebhookProcessor', () => {
it.each([
['deleted', null],
['disabled', { id: 'wh-1', active: false, url: 'https://8.8.8.8/hook', events: ['*'] }],
+ ['reassigned', { id: 'wh-1', active: true, sessionId: 'sess-2', url: 'https://8.8.8.8/hook', events: ['*'] }],
])('files nothing for a job whose webhook was %s', async (_label, row) => {
repo.findOne.mockResolvedValue(row);
diff --git a/src/modules/queue/processors/webhook.processor.ts b/src/modules/queue/processors/webhook.processor.ts
index fd4aadf96..2eccf4c4f 100644
--- a/src/modules/queue/processors/webhook.processor.ts
+++ b/src/modules/queue/processors/webhook.processor.ts
@@ -148,7 +148,7 @@ export class WebhookProcessor extends WorkerHost {
// headers and secret, as the reconciler's replay does, so a receiver move or a rotated
// secret or auth header applies to jobs already waiting in the queue.
// A read error lands in the catch below and counts as a failed attempt, like a failed POST.
- const current = await this.loadDeliverableWebhook(webhookId, event);
+ const current = await this.loadDeliverableWebhook(webhookId, event, sessionId);
if (!current) {
this.logger.warn('Skipping queued webhook delivery: webhook removed, disabled or unsubscribed', {
webhookId,
@@ -190,9 +190,9 @@ export class WebhookProcessor extends WorkerHost {
}
/** The webhook row as it is now, or null when it was deleted, disabled or no longer takes `event`. */
- private async loadDeliverableWebhook(webhookId: string, event: string): Promise<Webhook | null> {
+ private async loadDeliverableWebhook(webhookId: string, event: string, sessionId: string): Promise<Webhook | null> {
const row = await this.webhookRepository.findOne({ where: { id: webhookId } });
- return isDeliverableWebhook(row, event) ? row : null;
+ return isDeliverableWebhook(row, event, sessionId) ? row : null;
}
/**
@@ -360,7 +360,7 @@ export class WebhookProcessor extends WorkerHost {
// record against the enqueue-time snapshot rather than lose the failure.
let url = job.data.url;
try {
- const current = await this.loadDeliverableWebhook(webhookId, event);
+ const current = await this.loadDeliverableWebhook(webhookId, event, sessionId);
if (!current) {
return;
}
diff --git a/src/modules/webhook/utils/deliver-once.ts b/src/modules/webhook/utils/deliver-once.ts
index 008636f0b..072338516 100644
--- a/src/modules/webhook/utils/deliver-once.ts
+++ b/src/modules/webhook/utils/deliver-once.ts
@@ -33,11 +33,18 @@ export function sanitizeCustomHeaders(custom: Record<string, string> | null | un
* moment (a queued job, a direct retry, an outbox replay) runs this against a fresh row, so a removed,
* disabled or unsubscribed webhook stops receiving the event.
*/
-export function isDeliverableWebhook<T extends { active: boolean; events: string[] }>(
+export function isDeliverableWebhook<T extends { active: boolean; events: string[]; sessionId?: string }>(
row: T | null | undefined,
event: string,
+ sessionId?: string,
): row is T {
- return !!row && row.active && Array.isArray(row.events) && (row.events.includes(event) || row.events.includes('*'));
+ return (
+ !!row &&
+ (sessionId === undefined || row.sessionId === sessionId) &&
+ row.active &&
+ Array.isArray(row.events) &&
+ (row.events.includes(event) || row.events.includes('*'))
+ );
}
/** HMAC-SHA256 over the exact pre-serialized body, prefixed for receiver-side verification. */
diff --git a/src/modules/webhook/webhook-delivery.service.spec.ts b/src/modules/webhook/webhook-delivery.service.spec.ts
index 9d8496869..04a0f039c 100644
--- a/src/modules/webhook/webhook-delivery.service.spec.ts
+++ b/src/modules/webhook/webhook-delivery.service.spec.ts
@@ -38,8 +38,9 @@ import { QUEUE_NAMES } from '../queue/queue-names';
import { getWebhookDeliveryFailuresTotal } from '../../common/metrics/webhook-delivery-metrics';
import { ConcurrencyLimiter } from '../../common/utils/concurrency-limiter';
+const storedWebhooks = new Map<string, Webhook>();
function createMockWebhook(overrides: Partial<Webhook> = {}): Webhook {
- return {
+ const row = {
id: 'wh-uuid-1',
sessionId: 'sess-1',
url: 'https://example.com/webhook',
@@ -55,6 +56,8 @@ function createMockWebhook(overrides: Partial<Webhook> = {}): Webhook {
session: undefined as unknown as Session,
...overrides,
};
+ storedWebhooks.set(row.id, row);
+ return row;
}
// The recorder counts terminal rows with `attempts: MoreThan(0)`; honour that operator like the
@@ -79,14 +82,15 @@ describe('WebhookDeliveryService', () => {
let insertedFailures: Array<Partial<WebhookDeliveryFailure>>;
beforeEach(async () => {
+ storedWebhooks.clear();
repository = {
find: jest.fn(),
- // A direct retry re-reads its webhook. By default it reads back the row dispatch loaded, so a
- // test changes the webhook between attempts by overriding this.
- findOne: jest.fn().mockImplementation(async (opts: { where: { id: string } }) => {
- const rows = (await (repository.find as jest.Mock)()) as Webhook[] | undefined;
- return rows?.find(w => w.id === opts.where.id) ?? null;
- }),
+ // Read the seeded row independently of the dispatch subscription query.
+ findOne: jest
+ .fn()
+ .mockImplementation((opts: { where: { id: string } }) =>
+ Promise.resolve(storedWebhooks.get(opts.where.id) ?? null),
+ ),
update: jest.fn().mockResolvedValue({ affected: 1 }),
};
@@ -1860,7 +1864,7 @@ describe('WebhookDeliveryService', () => {
return def as T;
});
const webhook = createMockWebhook({ retryCount: 2 });
- (repository.findOne as jest.Mock).mockResolvedValue(null);
+ (repository.findOne as jest.Mock).mockResolvedValueOnce(webhook).mockResolvedValue(null);
webhookQueue.add.mockRejectedValueOnce(new Error('redis down'));
const mockFetch = undiciFetch as jest.Mock;
(hookManager.execute as jest.Mock).mockResolvedValue({ continue: true, data: {} });
@@ -1973,7 +1977,9 @@ describe('WebhookDeliveryService', () => {
['unsubscribed from the event', { events: ['message.ack'] }],
])('stops retrying, files nothing and reports cancelled when the webhook was %s', async (_label, change) => {
const webhook = createMockWebhook({ retryCount: 3 });
- (repository.findOne as jest.Mock).mockResolvedValue(change && { ...webhook, ...change });
+ (repository.findOne as jest.Mock)
+ .mockResolvedValueOnce(webhook)
+ .mockResolvedValue(change && { ...webhook, ...change });
mockFetch.mockRejectedValue(new Error('receiver down'));
const failuresBefore = getWebhookDeliveryFailuresTotal();
@@ -1992,7 +1998,7 @@ describe('WebhookDeliveryService', () => {
it('sends a retry to the current url, signed with the current secret', async () => {
const webhook = createMockWebhook({ retryCount: 2, secret: 'old-secret-0123456789' });
const moved = { ...webhook, url: 'https://moved.example/hook', secret: 'new-secret-0123456789' };
- (repository.findOne as jest.Mock).mockResolvedValue(moved);
+ (repository.findOne as jest.Mock).mockResolvedValueOnce(webhook).mockResolvedValue(moved);
mockFetch.mockRejectedValueOnce(new Error('receiver down')).mockResolvedValueOnce({ ok: true, status: 200 });
await expect(service.redeliver(webhook, 'sess-1', 'message.received', 'k', {})).resolves.toBe('delivered');
@@ -2007,7 +2013,7 @@ describe('WebhookDeliveryService', () => {
it('drops the signature from a retry once the secret is cleared', async () => {
const webhook = createMockWebhook({ retryCount: 2, secret: 'old-secret-0123456789' });
- (repository.findOne as jest.Mock).mockResolvedValue({ ...webhook, secret: null });
+ (repository.findOne as jest.Mock).mockResolvedValueOnce(webhook).mockResolvedValue({ ...webhook, secret: null });
mockFetch.mockRejectedValueOnce(new Error('receiver down')).mockResolvedValueOnce({ ok: true, status: 200 });
await service.redeliver(webhook, 'sess-1', 'message.received', 'k', {});
@@ -2019,7 +2025,7 @@ describe('WebhookDeliveryService', () => {
it('counts a failed re-read as a failed attempt', async () => {
const webhook = createMockWebhook({ retryCount: 2 });
- (repository.findOne as jest.Mock).mockRejectedValue(new Error('db down'));
+ (repository.findOne as jest.Mock).mockResolvedValueOnce(webhook).mockRejectedValue(new Error('db down'));
mockFetch.mockRejectedValue(new Error('receiver down'));
await expect(service.redeliver(webhook, 'sess-1', 'message.received', 'k', {})).resolves.toBe('failed');
@@ -2113,12 +2119,14 @@ describe('WebhookDeliveryService', () => {
beforeEach(() => {
for (const key of Object.keys(hooksBySession)) delete hooksBySession[key];
(repository.find as jest.Mock).mockImplementation((opts: { where: { sessionId: string } }) =>
- Promise.resolve(hooksBySession[opts.where.sessionId] ?? []),
+ Promise.resolve(
+ (hooksBySession[opts.where.sessionId] ?? []).map(row => ({ ...row, sessionId: opts.where.sessionId })),
+ ),
);
(repository.findOne as jest.Mock).mockImplementation((opts: { where: { id: string } }) =>
Promise.resolve(
- Object.values(hooksBySession)
- .flat()
+ Object.entries(hooksBySession)
+ .flatMap(([sessionId, rows]) => rows.map(row => ({ ...row, sessionId })))
.find(w => w.id === opts.where.id) ?? null,
),
);
@@ -2306,6 +2314,7 @@ describe('WebhookDeliveryService', () => {
it('marks a webhook failing on a failed attempt', async () => {
const webhook = createMockWebhook({ id: 'wh-a', retryCount: 1 });
+ hooksBySession['sess-1'] = [webhook];
mockFetch.mockRejectedValue(new Error('receiver down'));
await service.redeliver(webhook, 'sess-1', 'message.received', 'k', {});
diff --git a/src/modules/webhook/webhook-delivery.service.ts b/src/modules/webhook/webhook-delivery.service.ts
index c61ddb80f..4e2d085b4 100644
--- a/src/modules/webhook/webhook-delivery.service.ts
+++ b/src/modules/webhook/webhook-delivery.service.ts
@@ -985,9 +985,9 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
* `body` is the pre-serialized payload from preflight, the exact bytes the size gate checked, so it
* is never re-serialized here.
*
- * Like a queued job, every retry re-reads the webhook row: a webhook removed, disabled or
+ * Like a queued job, every attempt re-reads the webhook row: a webhook removed, disabled or
* unsubscribed since the dispatch gets nothing more and no failure row (resolves false), and a
- * changed url, secret or header map applies from the next attempt. Retries back off exponentially
+ * changed url, secret or header map applies immediately. Retries back off exponentially
* (retryDelay, then twice that, and so on), the schedule BullMQ applies to a queued job.
*/
private async deliverWebhook(
@@ -1001,22 +1001,20 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
let current = webhook;
for (let attempt = 1; ; attempt++) {
try {
- if (attempt > 1) {
- // Inside the try: a read error counts as a failed attempt, as it does for a queued job.
- const row = await this.webhookRepository.findOne({ where: { id: webhook.id } });
- if (!isDeliverableWebhook(row, payload.event)) {
- this.logger.warn('Skipping webhook retry: webhook removed, disabled or unsubscribed', {
- webhookId: webhook.id,
- event: payload.event,
- deliveryId: payload.deliveryId,
- idempotencyKey: payload.idempotencyKey,
- action: 'webhook_skipped_stale',
- });
- this.failingWebhooks.delete(webhook.id);
- return false;
- }
- current = row;
+ // Inside the try: a read error counts as a failed attempt, as it does for a queued job.
+ const row = await this.webhookRepository.findOne({ where: { id: webhook.id } });
+ if (!isDeliverableWebhook(row, payload.event, payload.sessionId)) {
+ this.logger.warn('Skipping webhook delivery: webhook removed, disabled, unsubscribed or reassigned', {
+ webhookId: webhook.id,
+ event: payload.event,
+ deliveryId: payload.deliveryId,
+ idempotencyKey: payload.idempotencyKey,
+ action: 'webhook_skipped_stale',
+ });
+ this.failingWebhooks.delete(webhook.id);
+ return false;
}
+ current = row;
const headers = buildDeliveryHeaders(
current,
payload.event,
diff --git a/src/modules/webhook/webhook-outbox.service.ts b/src/modules/webhook/webhook-outbox.service.ts
index a1fb83a9d..c23bb23ce 100644
--- a/src/modules/webhook/webhook-outbox.service.ts
+++ b/src/modules/webhook/webhook-outbox.service.ts
@@ -1,6 +1,6 @@
import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
-import { In, IsNull, LessThan, Not, Repository } from 'typeorm';
+import { And, In, IsNull, LessThan, Not, Raw, Repository } from 'typeorm';
import { WebhookOutboxEvent, WebhookOutboxState } from './entities/webhook-outbox-event.entity';
import { createLogger } from '../../common/services/logger.service';
import { isUniqueViolation } from '../../common/utils/db-errors';
@@ -138,10 +138,24 @@ export class WebhookOutboxService implements OnModuleInit, OnModuleDestroy {
}
/** Unsettled rows older than the staleness window; queued jobs are checked before replay. */
- async findStale(olderThan: Date, limit: number): Promise<ReplayableDelivery[]> {
+ async findStale(olderThan: Date, limit: number, afterId?: string): Promise<ReplayableDelivery[]> {
+ const pending = { state: In(['pending', 'queued']), payload: Not(IsNull()), createdAt: LessThan(olderThan) };
const rows = await this.outbox.find({
- where: { state: In(['pending', 'queued']), createdAt: LessThan(olderThan) },
- order: { createdAt: 'ASC' },
+ where: afterId
+ ? {
+ ...pending,
+ // Compare in SQL so PostgreSQL's sub-millisecond timestamps are never rounded through Date.
+ createdAt: And(
+ LessThan(olderThan),
+ Raw(
+ alias =>
+ `(${alias}, "id") > (SELECT "createdAt", "id" FROM "webhook_outbox_events" WHERE "id" = :afterId)`,
+ { afterId },
+ ),
+ ),
+ }
+ : pending,
+ order: { createdAt: 'ASC', id: 'ASC' },
take: limit,
});
// A pending row always carries its payload; the guard is for a row whose payload was retired by
diff --git a/src/modules/webhook/webhook-reconciler.service.spec.ts b/src/modules/webhook/webhook-reconciler.service.spec.ts
index 08bcad39f..d671ed63f 100644
--- a/src/modules/webhook/webhook-reconciler.service.spec.ts
+++ b/src/modules/webhook/webhook-reconciler.service.spec.ts
@@ -58,7 +58,9 @@ describe('WebhookReconcilerService', () => {
isQueueJobPending: jest.fn().mockResolvedValue(false),
isLocallyPending: jest.fn().mockReturnValue(false),
};
- webhooks = { findOne: jest.fn().mockResolvedValue({ id: 'wh-1', active: true, events: ['*'] }) };
+ webhooks = {
+ findOne: jest.fn().mockResolvedValue({ id: 'wh-1', sessionId: 'sess-1', active: true, events: ['*'] }),
+ };
service = new WebhookReconcilerService(webhooks as never, outbox as never, delivery as never);
});
@@ -70,7 +72,7 @@ describe('WebhookReconcilerService', () => {
// Deriving a fresh key would make the replay read as a second event at the receiver rather than
// a retry of the first, which is the whole reason the key is stored rather than recomputed.
expect(delivery.redeliver).toHaveBeenCalledWith(
- { id: 'wh-1', active: true, events: ['*'] },
+ { id: 'wh-1', sessionId: 'sess-1', active: true, events: ['*'] },
'sess-1',
'message.received',
'stored-key_wh-1',
diff --git a/src/modules/webhook/webhook-reconciler.service.ts b/src/modules/webhook/webhook-reconciler.service.ts
index 1ab49cc2c..f4a9729dc 100644
--- a/src/modules/webhook/webhook-reconciler.service.ts
+++ b/src/modules/webhook/webhook-reconciler.service.ts
@@ -61,6 +61,7 @@ export class WebhookReconcilerService implements OnModuleInit, OnModuleDestroy {
private readonly logger = createLogger('WebhookReconcilerService');
private timer?: ReturnType<typeof setInterval>;
private sweeping = false;
+ private cursor?: string;
constructor(
@InjectRepository(Webhook, 'data') private readonly webhooks: Repository<Webhook>,
@@ -92,7 +93,14 @@ export class WebhookReconcilerService implements OnModuleInit, OnModuleDestroy {
if (this.sweeping) return stats;
this.sweeping = true;
try {
- const rows = await this.outbox.findStale(new Date(now.getTime() - opts.graceMs), opts.batchSize);
+ const cutoff = new Date(now.getTime() - opts.graceMs);
+ let rows = await this.outbox.findStale(cutoff, opts.batchSize, this.cursor);
+ if (rows.length === 0 && this.cursor) {
+ this.cursor = undefined;
+ rows = await this.outbox.findStale(cutoff, opts.batchSize);
+ }
+ // Live jobs may occupy a whole page. Advance past them without spending their replay budget.
+ this.cursor = rows.at(-1)?.id;
stats.scanned = rows.length;
for (const row of rows) {
if (this.delivery.isLocallyPending(row.idempotencyKey)) {
@@ -103,7 +111,7 @@ export class WebhookReconcilerService implements OnModuleInit, OnModuleDestroy {
continue;
}
const webhook = await this.webhooks.findOne({ where: { id: row.webhookId } });
- if (!isDeliverableWebhook(webhook, row.event)) {
+ if (!isDeliverableWebhook(webhook, row.event, row.sessionId)) {
// The subscription is gone, switched off or no longer lists this event; replaying it would
// deliver an event the operator has already unsubscribed from. Same test as the queue
// processor applies before every attempt.
diff --git a/src/modules/webhook/webhook-recovery-boundaries.spec.ts b/src/modules/webhook/webhook-recovery-boundaries.spec.ts
new file mode 100644
index 000000000..3c4e78080
--- /dev/null
+++ b/src/modules/webhook/webhook-recovery-boundaries.spec.ts
@@ -0,0 +1,179 @@
+import { ConfigService } from '@nestjs/config';
+import { createHmac, randomUUID } from 'node:crypto';
+import { DataSource, Repository } from 'typeorm';
+import { Response } from 'undici';
+import { HookManager } from '../../core/hooks';
+import * as ssrf from '../../common/security/ssrf-guard';
+import { Session } from '../session/entities/session.entity';
+import { Webhook } from './entities/webhook.entity';
+import { WebhookDeliveryFailure } from './entities/webhook-delivery-failure.entity';
+import { WebhookOutboxEvent } from './entities/webhook-outbox-event.entity';
+import { WebhookOutboxService } from './webhook-outbox.service';
+import { WebhookReconcilerService } from './webhook-reconciler.service';
+import { WebhookRedriveService } from './webhook-redrive.service';
+import { WebhookDeliveryService } from './webhook-delivery.service';
+
+describe('webhook recovery across live ownership and configuration changes', () => {
+ let ds: DataSource;
+ let webhooks: Repository<Webhook>;
+ let failures: Repository<WebhookDeliveryFailure>;
+ let primary: Webhook;
+ beforeEach(async () => {
+ ds = await new DataSource({
+ type: 'better-sqlite3',
+ database: ':memory:',
+ entities: [Session, Webhook, WebhookDeliveryFailure, WebhookOutboxEvent],
+ synchronize: true,
+ }).initialize();
+ for (const id of ['s1', 's2']) await ds.getRepository(Session).save({ id, name: id });
+ webhooks = ds.getRepository(Webhook);
+ failures = ds.getRepository(WebhookDeliveryFailure);
+ primary = await webhooks.save({
+ sessionId: 's1',
+ url: 'https://old.example',
+ events: ['message.received'],
+ active: true,
+ retryCount: 1,
+ });
+ });
+ afterEach(async () => {
+ jest.restoreAllMocks();
+ await ds.destroy();
+ });
+
+ it('walks past live jobs, handles equal timestamps, and wraps after a deleted cursor row', async () => {
+ const repository = ds.getRepository(WebhookOutboxEvent);
+ const outbox = new WebhookOutboxService(repository);
+ for (let i = 0; i < 5; i++) {
+ await outbox.open({
+ webhookId: primary.id,
+ sessionId: primary.sessionId,
+ event: 'message.received',
+ idempotencyKey: `key${i}`,
+ deliveryId: `job${i}`,
+ payload: { id: i },
+ });
+ await repository.update({ idempotencyKey: `key${i}` }, { id: `row${i}`, createdAt: new Date('2026-10-01') });
+ if (i !== 4) await outbox.markQueued(primary.id, `key${i}`, `job${i}`);
+ }
+ const delivery = {
+ isLocallyPending: jest.fn((key: string) => key === 'key0'),
+ isQueueJobPending: jest.fn((id: string) => Promise.resolve(id === 'job1' || id === 'job2')),
+ redeliver: jest.fn().mockResolvedValue('delivered'),
+ recordReplayExhaustion: jest.fn().mockResolvedValue(true),
+ };
+ const reconciler = new WebhookReconcilerService(webhooks, outbox, delivery as never);
+ const options = { intervalMs: 1000, graceMs: 0, batchSize: 2, maxAttempts: 5 };
+ for (let i = 0; i < 3; i++) expect((await reconciler.sweep(options)).scanned).toBeLessThanOrEqual(2);
+ expect(delivery.redeliver.mock.calls.map((call: unknown[]) => call[3])).toEqual(['key3', 'key4']);
+ await repository.delete('row4');
+ // A newly stranded row earlier than the cursor becomes reachable on the next cycle.
+ delivery.isLocallyPending.mockReturnValue(false);
+ await reconciler.sweep(options);
+ expect(delivery.redeliver.mock.calls.map((call: unknown[]) => call[3])).toContain('key0');
+ expect((await repository.findOneByOrFail({ id: 'row3' })).state).toBe('dispatched');
+ });
+
+ it.each(['disable', 'delete', 'unsubscribe', 'move-session', 'rotate'] as const)(
+ 'uses the current receiver for each redrive after a %s',
+ async change => {
+ for (let i = 0; i < 5; i++)
+ await failures.save({
+ webhookId: primary.id,
+ sessionId: primary.sessionId,
+ event: 'message.received',
+ url: primary.url,
+ idempotencyKey: `key${i}`,
+ deliveryId: randomUUID(),
+ attempts: 1,
+ lastError: 'HTTP 503',
+ payload: { id: i },
+ createdAt: new Date(Date.now() - 10000 + i),
+ });
+ const config = new ConfigService({ webhook: { failurePayloadRetentionHours: 24, timeout: 1000 } });
+ const delivery = new WebhookDeliveryService(
+ webhooks,
+ failures,
+ config,
+ new HookManager(),
+ new WebhookOutboxService(ds.getRepository(WebhookOutboxEvent)),
+ );
+ const redrive = new WebhookRedriveService(webhooks, failures, delivery, config);
+ let release!: () => void;
+ const held = new Promise<void>(resolve => (release = resolve));
+ let called = 0;
+ const post = jest.spyOn(ssrf, 'withSafeFetch').mockImplementation(async (_url, _init, use) => {
+ called++;
+ if (called <= 4) await held;
+ return use(new Response(null, { status: 200 }));
+ });
+ const running = redrive.redrive({});
+ for (let i = 0; i < 100 && called < 4; i++) await new Promise(resolve => setImmediate(resolve));
+ expect(called).toBe(4);
+ if (change === 'delete') await webhooks.delete(primary.id);
+ else
+ await webhooks.update(
+ primary.id,
+ change === 'disable'
+ ? { active: false }
+ : change === 'unsubscribe'
+ ? { events: ['message.ack'] }
+ : change === 'move-session'
+ ? { sessionId: 's2' }
+ : { url: 'https://new.example', secret: 'new-secret', headers: { Authorization: 'new-token' } },
+ );
+ release();
+ expect(await running).toMatchObject(
+ change === 'rotate' ? { delivered: 5, skipped: 0 } : { delivered: 4, skipped: 1 },
+ );
+ if (change === 'rotate') {
+ const [url, init] = post.mock.calls[4];
+ expect(typeof init.body).toBe('string');
+ expect(url).toBe('https://new.example');
+ expect(init.headers).toMatchObject({
+ Authorization: 'new-token',
+ 'X-OpenWA-Signature': `sha256=${createHmac('sha256', 'new-secret')
+ .update(init.body as string)
+ .digest('hex')}`,
+ });
+ } else {
+ expect(post).toHaveBeenCalledTimes(4);
+ expect(await failures.count()).toBe(1);
+ }
+ },
+ );
+
+ it('rechecks the receiver after a held webhook hook before the first POST', async () => {
+ const config = new ConfigService({ webhook: { failurePayloadRetentionHours: 24 } });
+ const hooks = new HookManager();
+ let release!: () => void;
+ const held = new Promise<void>(resolve => (release = resolve));
+ jest.spyOn(hooks, 'execute').mockImplementation(async (_event, data) => {
+ await held;
+ return { continue: true, data };
+ });
+ const delivery = new WebhookDeliveryService(
+ webhooks,
+ failures,
+ config,
+ hooks,
+ new WebhookOutboxService(ds.getRepository(WebhookOutboxEvent)),
+ );
+ const post = jest
+ .spyOn(ssrf, 'withSafeFetch')
+ .mockImplementation((_url, _init, use) => Promise.resolve(use(new Response(null, { status: 200 }))));
+ const running = delivery.redeliver(
+ primary,
+ primary.sessionId,
+ 'message.received',
+ 'held-key',
+ {},
+ { singleAttempt: true },
+ );
+ await webhooks.update(primary.id, { active: false });
+ release();
+ expect(await running).toBe('cancelled');
+ expect(post).not.toHaveBeenCalled();
+ expect(await failures.count()).toBe(0);
+ });
+});
diff --git a/src/modules/webhook/webhook-recovery.pg.spec.ts b/src/modules/webhook/webhook-recovery.pg.spec.ts
index 36253b380..d4e7a9e15 100644
--- a/src/modules/webhook/webhook-recovery.pg.spec.ts
+++ b/src/modules/webhook/webhook-recovery.pg.spec.ts
@@ -155,6 +155,34 @@ describePostgres('webhook and media recovery on PostgreSQL', () => {
await expect(failures.save({ ...input })).rejects.toThrow();
});
+ it('advances outbox pages without rounding PostgreSQL timestamp precision', async () => {
+ const repository = ds.getRepository(WebhookOutboxEvent);
+ const outbox = new WebhookOutboxService(repository);
+ for (let i = 1; i <= 3; i++) {
+ const row = await repository.save({
+ webhookId: input.webhookId,
+ sessionId,
+ event: input.event,
+ idempotencyKey: `precision-${i}`,
+ deliveryId: `job-${i}`,
+ payload: { body: 'retained' },
+ state: 'queued',
+ attempts: 0,
+ });
+ await ds.query('UPDATE webhook_outbox_events SET "createdAt" = $1 WHERE id = $2', [
+ `2026-01-01 00:00:00.000${i}00`,
+ row.id,
+ ]);
+ }
+ let cursor: string | undefined;
+ for (let i = 1; i <= 3; i++) {
+ const rows = await outbox.findStale(new Date(), 1, cursor);
+ expect(rows.map(row => row.idempotencyKey)).toEqual([`precision-${i}`]);
+ cursor = rows[0].id;
+ }
+ expect(await outbox.findStale(new Date(), 1, cursor)).toEqual([]);
+ });
+
it('retains queued outbox data and never reopens a settled row', async () => {
const repository = ds.getRepository(WebhookOutboxEvent);
const outbox = new WebhookOutboxService(repository);