Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(webhooks): keep recovery deliveries current
Yudhi Armyndharis94bdf4402d83
diff --git a/src/modules/queue/processors/webhook.processor.spec.ts b/src/modules/queue/processors/webhook.processor.spec.tsindex ba4e99c84..d8951e701 100644--- a/src/modules/queue/processors/webhook.processor.spec.ts+++ b/src/modules/queue/processors/webhook.processor.spec.ts@@ -61,7 +61,13 @@ describe('WebhookProcessor', () => {repo = {update: jest.fn().mockResolvedValue({ affected: 1 }),// The live row matches makeJob()'s snapshot unless a test says otherwise.- findOne: jest.fn().mockResolvedValue({ id: 'wh-1', active: true, url: 'https://8.8.8.8/hook', events: ['*'] }),+ findOne: jest.fn().mockResolvedValue({+ id: 'wh-1',+ active: true,+ sessionId: 'sess-1',+ url: 'https://8.8.8.8/hook',+ events: ['*'],+ }),};// Stateful like the real table: the recorder counts existing rows for the delivery before it// inserts, so a constant would leave that guard unexercised here and let a duplicated row pass.@@ -173,7 +179,13 @@ describe('WebhookProcessor', () => {it('persists a durable delivery-failure record on the final attempt (with parsed HTTP status)', async () => {mockFetch.mockResolvedValue({ ok: false, status: 503, statusText: 'Service Unavailable' });- repo.findOne.mockResolvedValue({ id: 'wh-x', active: true, url: 'https://8.8.8.8/h', events: ['*'] });+ repo.findOne.mockResolvedValue({+ id: 'wh-x',+ active: true,+ sessionId: 'sess-1',+ url: 'https://8.8.8.8/h',+ events: ['*'],+ });await expect(processor.process(makeJob({ maxRetries: 3, webhookId: 'wh-x', url: 'https://8.8.8.8/h' }, 2)),@@ -300,7 +312,13 @@ describe('WebhookProcessor', () => {// message — the resolved internal IP is a recon oracle. The server-side logger.error keeps full detail.it('redacts the resolved internal IP from the webhook:error payload and DLQ row on an SSRF block', async () => {process.env.WEBHOOK_SSRF_PROTECT = 'true';- repo.findOne.mockResolvedValue({ id: 'wh-1', active: true, url: 'https://169.254.169.254/h', events: ['*'] });+ repo.findOne.mockResolvedValue({+ id: 'wh-1',+ active: true,+ sessionId: 'sess-1',+ url: 'https://169.254.169.254/h',+ events: ['*'],+ });// final attempt (attemptsMade=0, maxRetries=1 → 1 >= 1) so the hook + DLQ fireawait expect(processor.process(makeJob({ url: 'https://169.254.169.254/h', maxRetries: 1 }, 0))).rejects.toThrow();@@ -362,12 +380,13 @@ describe('WebhookProcessor', () => {// The job is a snapshot from enqueue time; the operator may have changed the webhook since.describe('stale snapshot', () => {- const live = { id: 'wh-1', active: true, url: 'https://8.8.8.8/hook', events: ['*'] };+ const live = { id: 'wh-1', active: true, sessionId: 'sess-1', url: 'https://8.8.8.8/hook', events: ['*'] };it.each([['deleted', null],['disabled', { ...live, active: false }],['unsubscribed', { ...live, events: ['message.sent'] }],+ ['reassigned', { ...live, sessionId: 'sess-2' }],])('does not POST, retry or dead-letter a job for a %s webhook', async (_label, row) => {repo.findOne.mockResolvedValue(row);mockFetch.mockResolvedValue({ ok: true, status: 200 });@@ -433,7 +452,13 @@ describe('WebhookProcessor', () => {it('records a dead-letter row, metric, and webhook:error for a job failed by a double stall', async () => {const failuresBefore = getWebhookDeliveryFailuresTotal();// Re-pointed since enqueue: the row names the URL a retry would have used, not the snapshot's.- repo.findOne.mockResolvedValue({ id: 'wh-stall', active: true, url: 'https://8.8.8.8/s', events: ['*'] });+ repo.findOne.mockResolvedValue({+ id: 'wh-stall',+ active: true,+ sessionId: 'sess-1',+ url: 'https://8.8.8.8/s',+ events: ['*'],+ });await processor.onWorkerFailed(makeJob({ webhookId: 'wh-stall', url: 'https://8.8.8.8/old' }, 1),@@ -462,6 +487,7 @@ describe('WebhookProcessor', () => {it.each([['deleted', null],['disabled', { id: 'wh-1', active: false, url: 'https://8.8.8.8/hook', events: ['*'] }],+ ['reassigned', { id: 'wh-1', active: true, sessionId: 'sess-2', url: 'https://8.8.8.8/hook', events: ['*'] }],])('files nothing for a job whose webhook was %s', async (_label, row) => {repo.findOne.mockResolvedValue(row);diff --git a/src/modules/queue/processors/webhook.processor.ts b/src/modules/queue/processors/webhook.processor.tsindex fd4aadf96..2eccf4c4f 100644--- a/src/modules/queue/processors/webhook.processor.ts+++ b/src/modules/queue/processors/webhook.processor.ts@@ -148,7 +148,7 @@ export class WebhookProcessor extends WorkerHost {// headers and secret, as the reconciler's replay does, so a receiver move or a rotated// secret or auth header applies to jobs already waiting in the queue.// A read error lands in the catch below and counts as a failed attempt, like a failed POST.- const current = await this.loadDeliverableWebhook(webhookId, event);+ const current = await this.loadDeliverableWebhook(webhookId, event, sessionId);if (!current) {this.logger.warn('Skipping queued webhook delivery: webhook removed, disabled or unsubscribed', {webhookId,@@ -190,9 +190,9 @@ export class WebhookProcessor extends WorkerHost {}/** The webhook row as it is now, or null when it was deleted, disabled or no longer takes `event`. */- private async loadDeliverableWebhook(webhookId: string, event: string): Promise<Webhook | null> {+ private async loadDeliverableWebhook(webhookId: string, event: string, sessionId: string): Promise<Webhook | null> {const row = await this.webhookRepository.findOne({ where: { id: webhookId } });- return isDeliverableWebhook(row, event) ? row : null;+ return isDeliverableWebhook(row, event, sessionId) ? row : null;}/**@@ -360,7 +360,7 @@ export class WebhookProcessor extends WorkerHost {// record against the enqueue-time snapshot rather than lose the failure.let url = job.data.url;try {- const current = await this.loadDeliverableWebhook(webhookId, event);+ const current = await this.loadDeliverableWebhook(webhookId, event, sessionId);if (!current) {return;}diff --git a/src/modules/webhook/utils/deliver-once.ts b/src/modules/webhook/utils/deliver-once.tsindex 008636f0b..072338516 100644--- a/src/modules/webhook/utils/deliver-once.ts+++ b/src/modules/webhook/utils/deliver-once.ts@@ -33,11 +33,18 @@ export function sanitizeCustomHeaders(custom: Record<string, string> | null | un* moment (a queued job, a direct retry, an outbox replay) runs this against a fresh row, so a removed,* disabled or unsubscribed webhook stops receiving the event.*/-export function isDeliverableWebhook<T extends { active: boolean; events: string[] }>(+export function isDeliverableWebhook<T extends { active: boolean; events: string[]; sessionId?: string }>(row: T | null | undefined,event: string,+ sessionId?: string,): row is T {- return !!row && row.active && Array.isArray(row.events) && (row.events.includes(event) || row.events.includes('*'));+ return (+ !!row &&+ (sessionId === undefined || row.sessionId === sessionId) &&+ row.active &&+ Array.isArray(row.events) &&+ (row.events.includes(event) || row.events.includes('*'))+ );}/** HMAC-SHA256 over the exact pre-serialized body, prefixed for receiver-side verification. */diff --git a/src/modules/webhook/webhook-delivery.service.spec.ts b/src/modules/webhook/webhook-delivery.service.spec.tsindex 9d8496869..04a0f039c 100644--- a/src/modules/webhook/webhook-delivery.service.spec.ts+++ b/src/modules/webhook/webhook-delivery.service.spec.ts@@ -38,8 +38,9 @@ import { QUEUE_NAMES } from '../queue/queue-names';import { getWebhookDeliveryFailuresTotal } from '../../common/metrics/webhook-delivery-metrics';import { ConcurrencyLimiter } from '../../common/utils/concurrency-limiter';+const storedWebhooks = new Map<string, Webhook>();function createMockWebhook(overrides: Partial<Webhook> = {}): Webhook {- return {+ const row = {id: 'wh-uuid-1',sessionId: 'sess-1',url: 'https://example.com/webhook',@@ -55,6 +56,8 @@ function createMockWebhook(overrides: Partial<Webhook> = {}): Webhook {session: undefined as unknown as Session,...overrides,};+ storedWebhooks.set(row.id, row);+ return row;}// The recorder counts terminal rows with `attempts: MoreThan(0)`; honour that operator like the@@ -79,14 +82,15 @@ describe('WebhookDeliveryService', () => {let insertedFailures: Array<Partial<WebhookDeliveryFailure>>;beforeEach(async () => {+ storedWebhooks.clear();repository = {find: jest.fn(),- // A direct retry re-reads its webhook. By default it reads back the row dispatch loaded, so a- // test changes the webhook between attempts by overriding this.- findOne: jest.fn().mockImplementation(async (opts: { where: { id: string } }) => {- const rows = (await (repository.find as jest.Mock)()) as Webhook[] | undefined;- return rows?.find(w => w.id === opts.where.id) ?? null;- }),+ // Read the seeded row independently of the dispatch subscription query.+ findOne: jest+ .fn()+ .mockImplementation((opts: { where: { id: string } }) =>+ Promise.resolve(storedWebhooks.get(opts.where.id) ?? null),+ ),update: jest.fn().mockResolvedValue({ affected: 1 }),};@@ -1860,7 +1864,7 @@ describe('WebhookDeliveryService', () => {return def as T;});const webhook = createMockWebhook({ retryCount: 2 });- (repository.findOne as jest.Mock).mockResolvedValue(null);+ (repository.findOne as jest.Mock).mockResolvedValueOnce(webhook).mockResolvedValue(null);webhookQueue.add.mockRejectedValueOnce(new Error('redis down'));const mockFetch = undiciFetch as jest.Mock;(hookManager.execute as jest.Mock).mockResolvedValue({ continue: true, data: {} });@@ -1973,7 +1977,9 @@ describe('WebhookDeliveryService', () => {['unsubscribed from the event', { events: ['message.ack'] }],])('stops retrying, files nothing and reports cancelled when the webhook was %s', async (_label, change) => {const webhook = createMockWebhook({ retryCount: 3 });- (repository.findOne as jest.Mock).mockResolvedValue(change && { ...webhook, ...change });+ (repository.findOne as jest.Mock)+ .mockResolvedValueOnce(webhook)+ .mockResolvedValue(change && { ...webhook, ...change });mockFetch.mockRejectedValue(new Error('receiver down'));const failuresBefore = getWebhookDeliveryFailuresTotal();@@ -1992,7 +1998,7 @@ describe('WebhookDeliveryService', () => {it('sends a retry to the current url, signed with the current secret', async () => {const webhook = createMockWebhook({ retryCount: 2, secret: 'old-secret-0123456789' });const moved = { ...webhook, url: 'https://moved.example/hook', secret: 'new-secret-0123456789' };- (repository.findOne as jest.Mock).mockResolvedValue(moved);+ (repository.findOne as jest.Mock).mockResolvedValueOnce(webhook).mockResolvedValue(moved);mockFetch.mockRejectedValueOnce(new Error('receiver down')).mockResolvedValueOnce({ ok: true, status: 200 });await expect(service.redeliver(webhook, 'sess-1', 'message.received', 'k', {})).resolves.toBe('delivered');@@ -2007,7 +2013,7 @@ describe('WebhookDeliveryService', () => {it('drops the signature from a retry once the secret is cleared', async () => {const webhook = createMockWebhook({ retryCount: 2, secret: 'old-secret-0123456789' });- (repository.findOne as jest.Mock).mockResolvedValue({ ...webhook, secret: null });+ (repository.findOne as jest.Mock).mockResolvedValueOnce(webhook).mockResolvedValue({ ...webhook, secret: null });mockFetch.mockRejectedValueOnce(new Error('receiver down')).mockResolvedValueOnce({ ok: true, status: 200 });await service.redeliver(webhook, 'sess-1', 'message.received', 'k', {});@@ -2019,7 +2025,7 @@ describe('WebhookDeliveryService', () => {it('counts a failed re-read as a failed attempt', async () => {const webhook = createMockWebhook({ retryCount: 2 });- (repository.findOne as jest.Mock).mockRejectedValue(new Error('db down'));+ (repository.findOne as jest.Mock).mockResolvedValueOnce(webhook).mockRejectedValue(new Error('db down'));mockFetch.mockRejectedValue(new Error('receiver down'));await expect(service.redeliver(webhook, 'sess-1', 'message.received', 'k', {})).resolves.toBe('failed');@@ -2113,12 +2119,14 @@ describe('WebhookDeliveryService', () => {beforeEach(() => {for (const key of Object.keys(hooksBySession)) delete hooksBySession[key];(repository.find as jest.Mock).mockImplementation((opts: { where: { sessionId: string } }) =>- Promise.resolve(hooksBySession[opts.where.sessionId] ?? []),+ Promise.resolve(+ (hooksBySession[opts.where.sessionId] ?? []).map(row => ({ ...row, sessionId: opts.where.sessionId })),+ ),);(repository.findOne as jest.Mock).mockImplementation((opts: { where: { id: string } }) =>Promise.resolve(- Object.values(hooksBySession)- .flat()+ Object.entries(hooksBySession)+ .flatMap(([sessionId, rows]) => rows.map(row => ({ ...row, sessionId }))).find(w => w.id === opts.where.id) ?? null,),);@@ -2306,6 +2314,7 @@ describe('WebhookDeliveryService', () => {it('marks a webhook failing on a failed attempt', async () => {const webhook = createMockWebhook({ id: 'wh-a', retryCount: 1 });+ hooksBySession['sess-1'] = [webhook];mockFetch.mockRejectedValue(new Error('receiver down'));await service.redeliver(webhook, 'sess-1', 'message.received', 'k', {});diff --git a/src/modules/webhook/webhook-delivery.service.ts b/src/modules/webhook/webhook-delivery.service.tsindex c61ddb80f..4e2d085b4 100644--- a/src/modules/webhook/webhook-delivery.service.ts+++ b/src/modules/webhook/webhook-delivery.service.ts@@ -985,9 +985,9 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {* `body` is the pre-serialized payload from preflight, the exact bytes the size gate checked, so it* is never re-serialized here.*- * Like a queued job, every retry re-reads the webhook row: a webhook removed, disabled or+ * Like a queued job, every attempt re-reads the webhook row: a webhook removed, disabled or* unsubscribed since the dispatch gets nothing more and no failure row (resolves false), and a- * changed url, secret or header map applies from the next attempt. Retries back off exponentially+ * changed url, secret or header map applies immediately. Retries back off exponentially* (retryDelay, then twice that, and so on), the schedule BullMQ applies to a queued job.*/private async deliverWebhook(@@ -1001,22 +1001,20 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {let current = webhook;for (let attempt = 1; ; attempt++) {try {- if (attempt > 1) {- // Inside the try: a read error counts as a failed attempt, as it does for a queued job.- const row = await this.webhookRepository.findOne({ where: { id: webhook.id } });- if (!isDeliverableWebhook(row, payload.event)) {- this.logger.warn('Skipping webhook retry: webhook removed, disabled or unsubscribed', {- webhookId: webhook.id,- event: payload.event,- deliveryId: payload.deliveryId,- idempotencyKey: payload.idempotencyKey,- action: 'webhook_skipped_stale',- });- this.failingWebhooks.delete(webhook.id);- return false;- }- current = row;+ // Inside the try: a read error counts as a failed attempt, as it does for a queued job.+ const row = await this.webhookRepository.findOne({ where: { id: webhook.id } });+ if (!isDeliverableWebhook(row, payload.event, payload.sessionId)) {+ this.logger.warn('Skipping webhook delivery: webhook removed, disabled, unsubscribed or reassigned', {+ webhookId: webhook.id,+ event: payload.event,+ deliveryId: payload.deliveryId,+ idempotencyKey: payload.idempotencyKey,+ action: 'webhook_skipped_stale',+ });+ this.failingWebhooks.delete(webhook.id);+ return false;}+ current = row;const headers = buildDeliveryHeaders(current,payload.event,diff --git a/src/modules/webhook/webhook-outbox.service.ts b/src/modules/webhook/webhook-outbox.service.tsindex a1fb83a9d..c23bb23ce 100644--- a/src/modules/webhook/webhook-outbox.service.ts+++ b/src/modules/webhook/webhook-outbox.service.ts@@ -1,6 +1,6 @@import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';-import { In, IsNull, LessThan, Not, Repository } from 'typeorm';+import { And, In, IsNull, LessThan, Not, Raw, Repository } from 'typeorm';import { WebhookOutboxEvent, WebhookOutboxState } from './entities/webhook-outbox-event.entity';import { createLogger } from '../../common/services/logger.service';import { isUniqueViolation } from '../../common/utils/db-errors';@@ -138,10 +138,24 @@ export class WebhookOutboxService implements OnModuleInit, OnModuleDestroy {}/** Unsettled rows older than the staleness window; queued jobs are checked before replay. */- async findStale(olderThan: Date, limit: number): Promise<ReplayableDelivery[]> {+ async findStale(olderThan: Date, limit: number, afterId?: string): Promise<ReplayableDelivery[]> {+ const pending = { state: In(['pending', 'queued']), payload: Not(IsNull()), createdAt: LessThan(olderThan) };const rows = await this.outbox.find({- where: { state: In(['pending', 'queued']), createdAt: LessThan(olderThan) },- order: { createdAt: 'ASC' },+ where: afterId+ ? {+ ...pending,+ // Compare in SQL so PostgreSQL's sub-millisecond timestamps are never rounded through Date.+ createdAt: And(+ LessThan(olderThan),+ Raw(+ alias =>+ `(${alias}, "id") > (SELECT "createdAt", "id" FROM "webhook_outbox_events" WHERE "id" = :afterId)`,+ { afterId },+ ),+ ),+ }+ : pending,+ order: { createdAt: 'ASC', id: 'ASC' },take: limit,});// A pending row always carries its payload; the guard is for a row whose payload was retired bydiff --git a/src/modules/webhook/webhook-reconciler.service.spec.ts b/src/modules/webhook/webhook-reconciler.service.spec.tsindex 08bcad39f..d671ed63f 100644--- a/src/modules/webhook/webhook-reconciler.service.spec.ts+++ b/src/modules/webhook/webhook-reconciler.service.spec.ts@@ -58,7 +58,9 @@ describe('WebhookReconcilerService', () => {isQueueJobPending: jest.fn().mockResolvedValue(false),isLocallyPending: jest.fn().mockReturnValue(false),};- webhooks = { findOne: jest.fn().mockResolvedValue({ id: 'wh-1', active: true, events: ['*'] }) };+ webhooks = {+ findOne: jest.fn().mockResolvedValue({ id: 'wh-1', sessionId: 'sess-1', active: true, events: ['*'] }),+ };service = new WebhookReconcilerService(webhooks as never, outbox as never, delivery as never);});@@ -70,7 +72,7 @@ describe('WebhookReconcilerService', () => {// Deriving a fresh key would make the replay read as a second event at the receiver rather than// a retry of the first, which is the whole reason the key is stored rather than recomputed.expect(delivery.redeliver).toHaveBeenCalledWith(- { id: 'wh-1', active: true, events: ['*'] },+ { id: 'wh-1', sessionId: 'sess-1', active: true, events: ['*'] },'sess-1','message.received','stored-key_wh-1',diff --git a/src/modules/webhook/webhook-reconciler.service.ts b/src/modules/webhook/webhook-reconciler.service.tsindex 1ab49cc2c..f4a9729dc 100644--- a/src/modules/webhook/webhook-reconciler.service.ts+++ b/src/modules/webhook/webhook-reconciler.service.ts@@ -61,6 +61,7 @@ export class WebhookReconcilerService implements OnModuleInit, OnModuleDestroy {private readonly logger = createLogger('WebhookReconcilerService');private timer?: ReturnType<typeof setInterval>;private sweeping = false;+ private cursor?: string;constructor(@InjectRepository(Webhook, 'data') private readonly webhooks: Repository<Webhook>,@@ -92,7 +93,14 @@ export class WebhookReconcilerService implements OnModuleInit, OnModuleDestroy {if (this.sweeping) return stats;this.sweeping = true;try {- const rows = await this.outbox.findStale(new Date(now.getTime() - opts.graceMs), opts.batchSize);+ const cutoff = new Date(now.getTime() - opts.graceMs);+ let rows = await this.outbox.findStale(cutoff, opts.batchSize, this.cursor);+ if (rows.length === 0 && this.cursor) {+ this.cursor = undefined;+ rows = await this.outbox.findStale(cutoff, opts.batchSize);+ }+ // Live jobs may occupy a whole page. Advance past them without spending their replay budget.+ this.cursor = rows.at(-1)?.id;stats.scanned = rows.length;for (const row of rows) {if (this.delivery.isLocallyPending(row.idempotencyKey)) {@@ -103,7 +111,7 @@ export class WebhookReconcilerService implements OnModuleInit, OnModuleDestroy {continue;}const webhook = await this.webhooks.findOne({ where: { id: row.webhookId } });- if (!isDeliverableWebhook(webhook, row.event)) {+ if (!isDeliverableWebhook(webhook, row.event, row.sessionId)) {// The subscription is gone, switched off or no longer lists this event; replaying it would// deliver an event the operator has already unsubscribed from. Same test as the queue// processor applies before every attempt.diff --git a/src/modules/webhook/webhook-recovery-boundaries.spec.ts b/src/modules/webhook/webhook-recovery-boundaries.spec.tsnew file mode 100644index 000000000..3c4e78080--- /dev/null+++ b/src/modules/webhook/webhook-recovery-boundaries.spec.ts@@ -0,0 +1,179 @@+import { ConfigService } from '@nestjs/config';+import { createHmac, randomUUID } from 'node:crypto';+import { DataSource, Repository } from 'typeorm';+import { Response } from 'undici';+import { HookManager } from '../../core/hooks';+import * as ssrf from '../../common/security/ssrf-guard';+import { Session } from '../session/entities/session.entity';+import { Webhook } from './entities/webhook.entity';+import { WebhookDeliveryFailure } from './entities/webhook-delivery-failure.entity';+import { WebhookOutboxEvent } from './entities/webhook-outbox-event.entity';+import { WebhookOutboxService } from './webhook-outbox.service';+import { WebhookReconcilerService } from './webhook-reconciler.service';+import { WebhookRedriveService } from './webhook-redrive.service';+import { WebhookDeliveryService } from './webhook-delivery.service';++describe('webhook recovery across live ownership and configuration changes', () => {+ let ds: DataSource;+ let webhooks: Repository<Webhook>;+ let failures: Repository<WebhookDeliveryFailure>;+ let primary: Webhook;+ beforeEach(async () => {+ ds = await new DataSource({+ type: 'better-sqlite3',+ database: ':memory:',+ entities: [Session, Webhook, WebhookDeliveryFailure, WebhookOutboxEvent],+ synchronize: true,+ }).initialize();+ for (const id of ['s1', 's2']) await ds.getRepository(Session).save({ id, name: id });+ webhooks = ds.getRepository(Webhook);+ failures = ds.getRepository(WebhookDeliveryFailure);+ primary = await webhooks.save({+ sessionId: 's1',+ url: 'https://old.example',+ events: ['message.received'],+ active: true,+ retryCount: 1,+ });+ });+ afterEach(async () => {+ jest.restoreAllMocks();+ await ds.destroy();+ });++ it('walks past live jobs, handles equal timestamps, and wraps after a deleted cursor row', async () => {+ const repository = ds.getRepository(WebhookOutboxEvent);+ const outbox = new WebhookOutboxService(repository);+ for (let i = 0; i < 5; i++) {+ await outbox.open({+ webhookId: primary.id,+ sessionId: primary.sessionId,+ event: 'message.received',+ idempotencyKey: `key${i}`,+ deliveryId: `job${i}`,+ payload: { id: i },+ });+ await repository.update({ idempotencyKey: `key${i}` }, { id: `row${i}`, createdAt: new Date('2026-10-01') });+ if (i !== 4) await outbox.markQueued(primary.id, `key${i}`, `job${i}`);+ }+ const delivery = {+ isLocallyPending: jest.fn((key: string) => key === 'key0'),+ isQueueJobPending: jest.fn((id: string) => Promise.resolve(id === 'job1' || id === 'job2')),+ redeliver: jest.fn().mockResolvedValue('delivered'),+ recordReplayExhaustion: jest.fn().mockResolvedValue(true),+ };+ const reconciler = new WebhookReconcilerService(webhooks, outbox, delivery as never);+ const options = { intervalMs: 1000, graceMs: 0, batchSize: 2, maxAttempts: 5 };+ for (let i = 0; i < 3; i++) expect((await reconciler.sweep(options)).scanned).toBeLessThanOrEqual(2);+ expect(delivery.redeliver.mock.calls.map((call: unknown[]) => call[3])).toEqual(['key3', 'key4']);+ await repository.delete('row4');+ // A newly stranded row earlier than the cursor becomes reachable on the next cycle.+ delivery.isLocallyPending.mockReturnValue(false);+ await reconciler.sweep(options);+ expect(delivery.redeliver.mock.calls.map((call: unknown[]) => call[3])).toContain('key0');+ expect((await repository.findOneByOrFail({ id: 'row3' })).state).toBe('dispatched');+ });++ it.each(['disable', 'delete', 'unsubscribe', 'move-session', 'rotate'] as const)(+ 'uses the current receiver for each redrive after a %s',+ async change => {+ for (let i = 0; i < 5; i++)+ await failures.save({+ webhookId: primary.id,+ sessionId: primary.sessionId,+ event: 'message.received',+ url: primary.url,+ idempotencyKey: `key${i}`,+ deliveryId: randomUUID(),+ attempts: 1,+ lastError: 'HTTP 503',+ payload: { id: i },+ createdAt: new Date(Date.now() - 10000 + i),+ });+ const config = new ConfigService({ webhook: { failurePayloadRetentionHours: 24, timeout: 1000 } });+ const delivery = new WebhookDeliveryService(+ webhooks,+ failures,+ config,+ new HookManager(),+ new WebhookOutboxService(ds.getRepository(WebhookOutboxEvent)),+ );+ const redrive = new WebhookRedriveService(webhooks, failures, delivery, config);+ let release!: () => void;+ const held = new Promise<void>(resolve => (release = resolve));+ let called = 0;+ const post = jest.spyOn(ssrf, 'withSafeFetch').mockImplementation(async (_url, _init, use) => {+ called++;+ if (called <= 4) await held;+ return use(new Response(null, { status: 200 }));+ });+ const running = redrive.redrive({});+ for (let i = 0; i < 100 && called < 4; i++) await new Promise(resolve => setImmediate(resolve));+ expect(called).toBe(4);+ if (change === 'delete') await webhooks.delete(primary.id);+ else+ await webhooks.update(+ primary.id,+ change === 'disable'+ ? { active: false }+ : change === 'unsubscribe'+ ? { events: ['message.ack'] }+ : change === 'move-session'+ ? { sessionId: 's2' }+ : { url: 'https://new.example', secret: 'new-secret', headers: { Authorization: 'new-token' } },+ );+ release();+ expect(await running).toMatchObject(+ change === 'rotate' ? { delivered: 5, skipped: 0 } : { delivered: 4, skipped: 1 },+ );+ if (change === 'rotate') {+ const [url, init] = post.mock.calls[4];+ expect(typeof init.body).toBe('string');+ expect(url).toBe('https://new.example');+ expect(init.headers).toMatchObject({+ Authorization: 'new-token',+ 'X-OpenWA-Signature': `sha256=${createHmac('sha256', 'new-secret')+ .update(init.body as string)+ .digest('hex')}`,+ });+ } else {+ expect(post).toHaveBeenCalledTimes(4);+ expect(await failures.count()).toBe(1);+ }+ },+ );++ it('rechecks the receiver after a held webhook hook before the first POST', async () => {+ const config = new ConfigService({ webhook: { failurePayloadRetentionHours: 24 } });+ const hooks = new HookManager();+ let release!: () => void;+ const held = new Promise<void>(resolve => (release = resolve));+ jest.spyOn(hooks, 'execute').mockImplementation(async (_event, data) => {+ await held;+ return { continue: true, data };+ });+ const delivery = new WebhookDeliveryService(+ webhooks,+ failures,+ config,+ hooks,+ new WebhookOutboxService(ds.getRepository(WebhookOutboxEvent)),+ );+ const post = jest+ .spyOn(ssrf, 'withSafeFetch')+ .mockImplementation((_url, _init, use) => Promise.resolve(use(new Response(null, { status: 200 }))));+ const running = delivery.redeliver(+ primary,+ primary.sessionId,+ 'message.received',+ 'held-key',+ {},+ { singleAttempt: true },+ );+ await webhooks.update(primary.id, { active: false });+ release();+ expect(await running).toBe('cancelled');+ expect(post).not.toHaveBeenCalled();+ expect(await failures.count()).toBe(0);+ });+});diff --git a/src/modules/webhook/webhook-recovery.pg.spec.ts b/src/modules/webhook/webhook-recovery.pg.spec.tsindex 36253b380..d4e7a9e15 100644--- a/src/modules/webhook/webhook-recovery.pg.spec.ts+++ b/src/modules/webhook/webhook-recovery.pg.spec.ts@@ -155,6 +155,34 @@ describePostgres('webhook and media recovery on PostgreSQL', () => {await expect(failures.save({ ...input })).rejects.toThrow();});+ it('advances outbox pages without rounding PostgreSQL timestamp precision', async () => {+ const repository = ds.getRepository(WebhookOutboxEvent);+ const outbox = new WebhookOutboxService(repository);+ for (let i = 1; i <= 3; i++) {+ const row = await repository.save({+ webhookId: input.webhookId,+ sessionId,+ event: input.event,+ idempotencyKey: `precision-${i}`,+ deliveryId: `job-${i}`,+ payload: { body: 'retained' },+ state: 'queued',+ attempts: 0,+ });+ await ds.query('UPDATE webhook_outbox_events SET "createdAt" = $1 WHERE id = $2', [+ `2026-01-01 00:00:00.000${i}00`,+ row.id,+ ]);+ }+ let cursor: string | undefined;+ for (let i = 1; i <= 3; i++) {+ const rows = await outbox.findStale(new Date(), 1, cursor);+ expect(rows.map(row => row.idempotencyKey)).toEqual([`precision-${i}`]);+ cursor = rows[0].id;+ }+ expect(await outbox.findStale(new Date(), 1, cursor)).toEqual([]);+ });+it('retains queued outbox data and never reopens a settled row', async () => {const repository = ds.getRepository(WebhookOutboxEvent);const outbox = new WebhookOutboxService(repository);