Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(webhook): support PostgreSQL recovery schemas

Yudhi Armyndharis89c21f288f20
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 2f95f072b..60ab4d62c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -250,20 +250,20 @@ jobs:
# differently between two statements, so an unstable order drops rows from a page walk), and
# that the data entities can build their own schema (raw SQL the ORM passes through verbatim,
# such as a partial index predicate, only case-folds on a real server), and that the UTC pin
- # holds across a backup round trip, a retention window and a local-midnight count. All five
+ # holds across a backup round trip, a retention window and a local-midnight count. All six
# self-skip unless DATABASE_TYPE=postgres, so they are a no-op in the default test job and only
# execute here against the postgres:16 service.
#
# TZ is deliberately NOT UTC: the pin's whole job is to make a `timestamp` column mean the same
# instant on a host that runs in another zone, and on a UTC runner every assertion about it
# would pass for the wrong reason (the UTC-pin spec fails outright rather than pass vacuously).
- - name: Postgres specs (FTS provider + boot-migration lock + list ordering + entity synchronize + UTC pin)
+ - name: Postgres specs (FTS + migrations + ordering + schema + UTC + recovery)
# --runInBand: the specs issue DDL against the one shared service database, and jest's
# default file-parallelism races their catalog writes (pg_type duplicate-key). The lock
# spec's own two-migrator race is in-file (Promise.all) and unaffected by runInBand. The
# synchronize spec additionally CREATEs and DROPs a scratch database of its own, which is
# only safe serialized.
- run: npx jest --runInBand src/database/migrations/__tests__/1782400000000-AddMessagesFts.pg.spec.ts src/database/migrations/__tests__/pg-boot-migrations.pg.spec.ts src/modules/message/message-list-ordering.pg.spec.ts src/database/migrations/__tests__/pg-entity-synchronize.pg.spec.ts src/database/postgres-utc.pg.spec.ts
+ run: npx jest --runInBand src/database/migrations/__tests__/1782400000000-AddMessagesFts.pg.spec.ts src/database/migrations/__tests__/pg-boot-migrations.pg.spec.ts src/modules/message/message-list-ordering.pg.spec.ts src/database/migrations/__tests__/pg-entity-synchronize.pg.spec.ts src/database/postgres-utc.pg.spec.ts src/modules/webhook/webhook-recovery.pg.spec.ts
env:
TZ: Asia/Jakarta
DATABASE_TYPE: postgres
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 256f410e8..7345107e8 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -229,17 +229,17 @@ jobs:
# database), that a paged message list has a total order, that the data entities can build
# their own schema (raw SQL the ORM passes through verbatim, such as a partial index predicate,
# only case-folds on a real server), and that the UTC pin holds across a backup round trip, a
- # retention window and a local-midnight count. All five self-skip unless DATABASE_TYPE=postgres,
+ # retention window and a local-midnight count. All six self-skip unless DATABASE_TYPE=postgres,
# so they are a no-op in the default test job and only execute here against the postgres:16
# service. TZ is deliberately not UTC, for the reason ci.yml states.
- - name: Postgres specs (FTS provider + boot-migration lock + list ordering + entity synchronize + UTC pin)
+ - name: Postgres specs (FTS + migrations + ordering + schema + UTC + recovery)
# --runInBand: same reason the CI job passes it — both specs issue DDL against the one
# shared service database, and jest's default file-parallelism races their catalog writes
# (the first v0.19.0 tag failed this step on `relation "messages" already exists` over a
# tree CI had passed minutes earlier). The lock spec's own two-migrator race is in-file
# (Promise.all) and unaffected by runInBand. The synchronize spec additionally CREATEs and
# DROPs a scratch database of its own, which is only safe serialized.
- run: npx jest --runInBand src/database/migrations/__tests__/1782400000000-AddMessagesFts.pg.spec.ts src/database/migrations/__tests__/pg-boot-migrations.pg.spec.ts src/modules/message/message-list-ordering.pg.spec.ts src/database/migrations/__tests__/pg-entity-synchronize.pg.spec.ts src/database/postgres-utc.pg.spec.ts
+ run: npx jest --runInBand src/database/migrations/__tests__/1782400000000-AddMessagesFts.pg.spec.ts src/database/migrations/__tests__/pg-boot-migrations.pg.spec.ts src/modules/message/message-list-ordering.pg.spec.ts src/database/migrations/__tests__/pg-entity-synchronize.pg.spec.ts src/database/postgres-utc.pg.spec.ts src/modules/webhook/webhook-recovery.pg.spec.ts
env:
TZ: Asia/Jakarta
DATABASE_TYPE: postgres
diff --git a/src/modules/webhook/webhook-recovery.pg.spec.ts b/src/modules/webhook/webhook-recovery.pg.spec.ts
new file mode 100644
index 000000000..ee54f05de
--- /dev/null
+++ b/src/modules/webhook/webhook-recovery.pg.spec.ts
@@ -0,0 +1,220 @@
+import { randomBytes } from 'crypto';
+import { ConfigService } from '@nestjs/config';
+import { DataSource, Repository } from 'typeorm';
+import { Session } from '../session/entities/session.entity';
+import { Message, MessageDirection, MessageStatus } from '../message/entities/message.entity';
+import { Webhook } from './entities/webhook.entity';
+import { WebhookDeliveryFailure } from './entities/webhook-delivery-failure.entity';
+import { WebhookOutboxEvent } from './entities/webhook-outbox-event.entity';
+import { WebhookOutboxService } from './webhook-outbox.service';
+import { WebhookRedriveService } from './webhook-redrive.service';
+import { recordWebhookDeliveryFailure } from './utils/record-delivery-failure';
+import { DeduplicateTerminalWebhookFailures1787200000000 } from '../../database/migrations/1787200000000-DeduplicateTerminalWebhookFailures';
+import { ChatMediaArchiveService } from '../chat-media/chat-media-archive.service';
+import type { StorageService } from '../../common/storage/storage.service';
+
+jest.mock('archiver', () => ({ default: jest.fn() }));
+const describePostgres = process.env.DATABASE_TYPE === 'postgres' ? describe : describe.skip;
+
+describePostgres('webhook and media recovery on PostgreSQL', () => {
+ let admin: DataSource;
+ let ds: DataSource;
+ let second: DataSource;
+ let failures: Repository<WebhookDeliveryFailure>;
+ let webhooks: Repository<Webhook>;
+ let sessionId: string;
+ const schema = `recovery_${process.pid}_${randomBytes(4).toString('hex')}`;
+ const logger = { error: jest.fn() };
+ const config = new ConfigService({ webhook: { failurePayloadRetentionHours: 24 } });
+ const input = {
+ sessionId: '',
+ webhookId: '',
+ event: 'message.received',
+ url: 'https://receiver.example',
+ idempotencyKey: 'key',
+ attempts: 3,
+ lastError: 'HTTP 503: unavailable',
+ };
+
+ beforeAll(async () => {
+ const options = {
+ type: 'postgres' as const,
+ host: process.env.DATABASE_HOST || 'localhost',
+ port: Number(process.env.DATABASE_PORT || 5432),
+ username: process.env.DATABASE_USERNAME || 'openwa',
+ password: process.env.DATABASE_PASSWORD || 'openwa',
+ database: process.env.DATABASE_NAME || 'openwa',
+ };
+ admin = await new DataSource(options).initialize();
+ await admin.query(`CREATE SCHEMA "${schema}"`);
+ const scoped = {
+ ...options,
+ schema,
+ extra: { options: `-c search_path=${schema},public` },
+ entities: [Session, Webhook, WebhookDeliveryFailure, WebhookOutboxEvent, Message],
+ };
+ ds = await new DataSource({ ...scoped, synchronize: true }).initialize();
+ second = await new DataSource(scoped).initialize();
+ failures = ds.getRepository(WebhookDeliveryFailure);
+ webhooks = ds.getRepository(Webhook);
+ sessionId = (await ds.getRepository(Session).save({ name: schema })).id;
+ input.sessionId = sessionId;
+ });
+ beforeEach(async () => {
+ await failures.clear();
+ await ds.getRepository(WebhookOutboxEvent).clear();
+ await webhooks.clear();
+ const webhook = await webhooks.save({ sessionId, url: input.url, events: ['message.received'], active: true });
+ input.webhookId = webhook.id;
+ });
+ afterEach(() => jest.restoreAllMocks());
+ afterAll(async () => {
+ if (second?.isInitialized) await second.destroy();
+ if (ds?.isInitialized) await ds.destroy();
+ if (admin?.isInitialized) {
+ await admin.query(`DROP SCHEMA "${schema}" CASCADE`);
+ await admin.destroy();
+ }
+ });
+
+ it('filters subscriptions before a batch and advances past failed deliveries', async () => {
+ await failures.save({
+ ...input,
+ webhookId: 'removed',
+ idempotencyKey: 'skipped',
+ payload: {},
+ createdAt: new Date(Date.now() - 3000),
+ });
+ await failures.save({
+ ...input,
+ idempotencyKey: 'bad',
+ payload: { body: 'bad' },
+ createdAt: new Date(Date.now() - 2000),
+ });
+ await failures.save({
+ ...input,
+ idempotencyKey: 'healthy',
+ payload: { body: 'healthy' },
+ createdAt: new Date(Date.now() - 1000),
+ });
+ await failures.save({ ...input, event: 'message.ack', idempotencyKey: 'unsubscribed', payload: {} });
+ await failures.save({
+ ...input,
+ idempotencyKey: 'expired',
+ payload: {},
+ createdAt: new Date(Date.now() - 25 * 3600000),
+ });
+ const delivery = {
+ redeliver: jest.fn(async (_webhook: Webhook, _session: string, _event: string, key: string) => {
+ if (key === 'bad') {
+ await recordWebhookDeliveryFailure(failures, logger, { ...input, idempotencyKey: key, attempts: 1 });
+ return 'failed';
+ }
+ await failures.delete({ idempotencyKey: key });
+ return 'delivered';
+ }),
+ };
+ const service = new WebhookRedriveService(webhooks, failures, delivery as never, config);
+ expect(await service.redrive({ limit: 1 }, [sessionId])).toMatchObject({ failed: 1, remaining: 2 });
+ expect(await service.redrive({ limit: 1 }, [sessionId])).toMatchObject({ delivered: 1, remaining: 1 });
+ expect(delivery.redeliver.mock.calls.map(call => call[3])).toEqual(['bad', 'healthy']);
+ const unsubscribed = await failures.findOneByOrFail({ idempotencyKey: 'unsubscribed' });
+ await webhooks.update({ id: input.webhookId }, { events: ['*'] });
+ expect(await service.redrive({ ids: [unsubscribed.id] })).toMatchObject({ delivered: 1, remaining: 0 });
+ const disabled = new WebhookRedriveService(
+ webhooks,
+ failures,
+ delivery as never,
+ new ConfigService({ webhook: { failurePayloadRetentionHours: 0 } }),
+ );
+ expect(await disabled.redrive({})).toMatchObject({ redriven: 0, remaining: 0 });
+
+ expect(await service.redrive({ sessionId: 'outside' }, [sessionId])).toMatchObject({ redriven: 0 });
+ });
+
+ it('converges two connections on one terminal identity', async () => {
+ const result = await Promise.all([
+ recordWebhookDeliveryFailure(failures, logger, input),
+ recordWebhookDeliveryFailure(second.getRepository(WebhookDeliveryFailure), logger, { ...input, attempts: 1 }),
+ ]);
+ expect(result.sort()).toEqual([false, true]);
+ expect(await failures.count()).toBe(1);
+ expect((await failures.findOneByOrFail({ idempotencyKey: 'key' })).attempts).toBe(4);
+ });
+
+ it('upgrades duplicate terminal records while preserving replay data', async () => {
+ await ds.query(`DROP INDEX "${schema}"."UQ_webhook_delivery_failures_terminal"`);
+ await failures.save({ ...input, payload: { body: 'retained' } });
+ await failures.save({ ...input, attempts: 5 });
+ const migration = new DeduplicateTerminalWebhookFailures1787200000000();
+ await migration.up(ds.createQueryRunner());
+ expect(await failures.count()).toBe(1);
+ const row = await failures.findOneOrFail({ where: { idempotencyKey: 'key' }, select: { id: true, payload: true } });
+ expect(row.payload).toEqual({ body: 'retained' });
+ await expect(failures.save({ ...input })).rejects.toThrow();
+ });
+
+ it('retains queued outbox data and never reopens a settled row', async () => {
+ const repository = ds.getRepository(WebhookOutboxEvent);
+ const outbox = new WebhookOutboxService(repository);
+ await outbox.open({ ...input, payload: { body: 'retained' }, deliveryId: 'job' });
+ await outbox.markQueued(input.webhookId, 'key', 'job');
+ await repository.update({ idempotencyKey: 'key' }, { createdAt: new Date(0) });
+ expect(await outbox.pruneSettled(1)).toBe(0);
+ expect(await outbox.findStale(new Date(), 1)).toEqual([
+ expect.objectContaining({ state: 'queued', payload: { body: 'retained' } }),
+ ]);
+ await outbox.close(input.webhookId, 'key', 'dispatched');
+ await outbox.markQueued(input.webhookId, 'key', 'late');
+ expect(await repository.findOneByOrFail({ idempotencyKey: 'key' })).toMatchObject({
+ state: 'dispatched',
+ payload: null,
+ });
+ });
+
+ it('publishes verified media atomically and preserves concurrent metadata', async () => {
+ const repository = ds.getRepository(Message);
+ const bytes = Buffer.from('verified media');
+ const storage = {
+ putFile: jest.fn().mockResolvedValue(undefined),
+ getFile: jest.fn().mockResolvedValue(bytes),
+ deleteFile: jest.fn().mockResolvedValue(undefined),
+ };
+ const archive = new ChatMediaArchiveService(
+ repository,
+ storage as unknown as StorageService,
+ new ConfigService({ chatMedia: { archiveEnabled: true, inlineMode: 'archive' } }),
+ );
+ const metadata = { media: { mimetype: 'image/png', data: bytes.toString('base64') } };
+ const base = {
+ sessionId,
+ chatId: '[email protected]',
+ body: '',
+ type: 'image',
+ direction: MessageDirection.INCOMING,
+ status: MessageStatus.SENT,
+ timestamp: 1,
+ metadata,
+ };
+ const first = await repository.save({ ...base, waMessageId: 'success' });
+ expect(await archive.archive(first)).not.toBeNull();
+ expect((await repository.findOneByOrFail({ id: first.id })).metadata?.media).toMatchObject({
+ archived: true,
+ omitted: true,
+ });
+ const raced = await repository.save({ ...base, waMessageId: 'race' });
+ const find = repository.findOne.bind(repository);
+ jest.spyOn(repository, 'findOne').mockImplementationOnce(async options => {
+ const snapshot = await find(options);
+ await repository.update({ id: raced.id }, { metadata: { ...metadata, reactions: ['kept'] } });
+ return snapshot;
+ });
+ expect(await archive.archive(raced)).toBeNull();
+ expect(await repository.findOneByOrFail({ id: raced.id })).toMatchObject({
+ mediaPath: null,
+ metadata: { ...metadata, reactions: ['kept'] },
+ });
+ });
+});
diff --git a/src/modules/webhook/webhook-redrive.service.ts b/src/modules/webhook/webhook-redrive.service.ts
index ff1d0ad1c..8ca552179 100644
--- a/src/modules/webhook/webhook-redrive.service.ts
+++ b/src/modules/webhook/webhook-redrive.service.ts
@@ -28,11 +28,11 @@ export interface WebhookRedriveRequest {
}
export interface WebhookRedriveResult {
- /** Rows replayed by this call: delivered plus enqueued. */
+ /** Rows delivered by this call. */
redriven: number;
/** Delivered by a direct POST; their failure rows are gone. */
delivered: number;
- /** Handed to the queue; the row is removed when the job delivers, and kept when it fails again. */
+ /** Reserved for compatibility; operator redrive always uses a direct POST and returns zero. */
enqueued: number;
/** The replay failed again; the row stays, with its attempt count raised. */
failed: number;
@@ -109,7 +109,11 @@ export class WebhookRedriveService {
this.failureRepository.manager.connection.options.type === 'postgres' ? 'subscribed.event' : 'subscribed.value';
const query = this.failureRepository
.createQueryBuilder('failure')
- .innerJoin(Webhook, 'webhook', 'webhook.id = failure.webhookId AND webhook.sessionId = failure.sessionId')
+ .innerJoin(
+ Webhook,
+ 'webhook',
+ 'CAST(webhook.id AS text) = failure.webhookId AND CAST(webhook.sessionId AS text) = failure.sessionId',
+ )
.where({
attempts: MoreThan(0),
payload: Not(IsNull()),