Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(webhook): support PostgreSQL recovery schemas
Yudhi Armyndharis89c21f288f20
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex 2f95f072b..60ab4d62c 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -250,20 +250,20 @@ jobs:# differently between two statements, so an unstable order drops rows from a page walk), and# that the data entities can build their own schema (raw SQL the ORM passes through verbatim,# such as a partial index predicate, only case-folds on a real server), and that the UTC pin- # holds across a backup round trip, a retention window and a local-midnight count. All five+ # holds across a backup round trip, a retention window and a local-midnight count. All six# self-skip unless DATABASE_TYPE=postgres, so they are a no-op in the default test job and only# execute here against the postgres:16 service.## TZ is deliberately NOT UTC: the pin's whole job is to make a `timestamp` column mean the same# instant on a host that runs in another zone, and on a UTC runner every assertion about it# would pass for the wrong reason (the UTC-pin spec fails outright rather than pass vacuously).- - name: Postgres specs (FTS provider + boot-migration lock + list ordering + entity synchronize + UTC pin)+ - name: Postgres specs (FTS + migrations + ordering + schema + UTC + recovery)# --runInBand: the specs issue DDL against the one shared service database, and jest's# default file-parallelism races their catalog writes (pg_type duplicate-key). The lock# spec's own two-migrator race is in-file (Promise.all) and unaffected by runInBand. The# synchronize spec additionally CREATEs and DROPs a scratch database of its own, which is# only safe serialized.- run: npx jest --runInBand src/database/migrations/__tests__/1782400000000-AddMessagesFts.pg.spec.ts src/database/migrations/__tests__/pg-boot-migrations.pg.spec.ts src/modules/message/message-list-ordering.pg.spec.ts src/database/migrations/__tests__/pg-entity-synchronize.pg.spec.ts src/database/postgres-utc.pg.spec.ts+ run: npx jest --runInBand src/database/migrations/__tests__/1782400000000-AddMessagesFts.pg.spec.ts src/database/migrations/__tests__/pg-boot-migrations.pg.spec.ts src/modules/message/message-list-ordering.pg.spec.ts src/database/migrations/__tests__/pg-entity-synchronize.pg.spec.ts src/database/postgres-utc.pg.spec.ts src/modules/webhook/webhook-recovery.pg.spec.tsenv:TZ: Asia/JakartaDATABASE_TYPE: postgresdiff --git a/.github/workflows/release.yml b/.github/workflows/release.ymlindex 256f410e8..7345107e8 100644--- a/.github/workflows/release.yml+++ b/.github/workflows/release.yml@@ -229,17 +229,17 @@ jobs:# database), that a paged message list has a total order, that the data entities can build# their own schema (raw SQL the ORM passes through verbatim, such as a partial index predicate,# only case-folds on a real server), and that the UTC pin holds across a backup round trip, a- # retention window and a local-midnight count. All five self-skip unless DATABASE_TYPE=postgres,+ # retention window and a local-midnight count. All six self-skip unless DATABASE_TYPE=postgres,# so they are a no-op in the default test job and only execute here against the postgres:16# service. TZ is deliberately not UTC, for the reason ci.yml states.- - name: Postgres specs (FTS provider + boot-migration lock + list ordering + entity synchronize + UTC pin)+ - name: Postgres specs (FTS + migrations + ordering + schema + UTC + recovery)# --runInBand: same reason the CI job passes it — both specs issue DDL against the one# shared service database, and jest's default file-parallelism races their catalog writes# (the first v0.19.0 tag failed this step on `relation "messages" already exists` over a# tree CI had passed minutes earlier). The lock spec's own two-migrator race is in-file# (Promise.all) and unaffected by runInBand. The synchronize spec additionally CREATEs and# DROPs a scratch database of its own, which is only safe serialized.- run: npx jest --runInBand src/database/migrations/__tests__/1782400000000-AddMessagesFts.pg.spec.ts src/database/migrations/__tests__/pg-boot-migrations.pg.spec.ts src/modules/message/message-list-ordering.pg.spec.ts src/database/migrations/__tests__/pg-entity-synchronize.pg.spec.ts src/database/postgres-utc.pg.spec.ts+ run: npx jest --runInBand src/database/migrations/__tests__/1782400000000-AddMessagesFts.pg.spec.ts src/database/migrations/__tests__/pg-boot-migrations.pg.spec.ts src/modules/message/message-list-ordering.pg.spec.ts src/database/migrations/__tests__/pg-entity-synchronize.pg.spec.ts src/database/postgres-utc.pg.spec.ts src/modules/webhook/webhook-recovery.pg.spec.tsenv:TZ: Asia/JakartaDATABASE_TYPE: postgresdiff --git a/src/modules/webhook/webhook-recovery.pg.spec.ts b/src/modules/webhook/webhook-recovery.pg.spec.tsnew file mode 100644index 000000000..ee54f05de--- /dev/null+++ b/src/modules/webhook/webhook-recovery.pg.spec.ts@@ -0,0 +1,220 @@+import { randomBytes } from 'crypto';+import { ConfigService } from '@nestjs/config';+import { DataSource, Repository } from 'typeorm';+import { Session } from '../session/entities/session.entity';+import { Message, MessageDirection, MessageStatus } from '../message/entities/message.entity';+import { Webhook } from './entities/webhook.entity';+import { WebhookDeliveryFailure } from './entities/webhook-delivery-failure.entity';+import { WebhookOutboxEvent } from './entities/webhook-outbox-event.entity';+import { WebhookOutboxService } from './webhook-outbox.service';+import { WebhookRedriveService } from './webhook-redrive.service';+import { recordWebhookDeliveryFailure } from './utils/record-delivery-failure';+import { DeduplicateTerminalWebhookFailures1787200000000 } from '../../database/migrations/1787200000000-DeduplicateTerminalWebhookFailures';+import { ChatMediaArchiveService } from '../chat-media/chat-media-archive.service';+import type { StorageService } from '../../common/storage/storage.service';++jest.mock('archiver', () => ({ default: jest.fn() }));+const describePostgres = process.env.DATABASE_TYPE === 'postgres' ? describe : describe.skip;++describePostgres('webhook and media recovery on PostgreSQL', () => {+ let admin: DataSource;+ let ds: DataSource;+ let second: DataSource;+ let failures: Repository<WebhookDeliveryFailure>;+ let webhooks: Repository<Webhook>;+ let sessionId: string;+ const schema = `recovery_${process.pid}_${randomBytes(4).toString('hex')}`;+ const logger = { error: jest.fn() };+ const config = new ConfigService({ webhook: { failurePayloadRetentionHours: 24 } });+ const input = {+ sessionId: '',+ webhookId: '',+ event: 'message.received',+ url: 'https://receiver.example',+ idempotencyKey: 'key',+ attempts: 3,+ lastError: 'HTTP 503: unavailable',+ };++ beforeAll(async () => {+ const options = {+ type: 'postgres' as const,+ host: process.env.DATABASE_HOST || 'localhost',+ port: Number(process.env.DATABASE_PORT || 5432),+ username: process.env.DATABASE_USERNAME || 'openwa',+ password: process.env.DATABASE_PASSWORD || 'openwa',+ database: process.env.DATABASE_NAME || 'openwa',+ };+ admin = await new DataSource(options).initialize();+ await admin.query(`CREATE SCHEMA "${schema}"`);+ const scoped = {+ ...options,+ schema,+ extra: { options: `-c search_path=${schema},public` },+ entities: [Session, Webhook, WebhookDeliveryFailure, WebhookOutboxEvent, Message],+ };+ ds = await new DataSource({ ...scoped, synchronize: true }).initialize();+ second = await new DataSource(scoped).initialize();+ failures = ds.getRepository(WebhookDeliveryFailure);+ webhooks = ds.getRepository(Webhook);+ sessionId = (await ds.getRepository(Session).save({ name: schema })).id;+ input.sessionId = sessionId;+ });+ beforeEach(async () => {+ await failures.clear();+ await ds.getRepository(WebhookOutboxEvent).clear();+ await webhooks.clear();+ const webhook = await webhooks.save({ sessionId, url: input.url, events: ['message.received'], active: true });+ input.webhookId = webhook.id;+ });+ afterEach(() => jest.restoreAllMocks());+ afterAll(async () => {+ if (second?.isInitialized) await second.destroy();+ if (ds?.isInitialized) await ds.destroy();+ if (admin?.isInitialized) {+ await admin.query(`DROP SCHEMA "${schema}" CASCADE`);+ await admin.destroy();+ }+ });++ it('filters subscriptions before a batch and advances past failed deliveries', async () => {+ await failures.save({+ ...input,+ webhookId: 'removed',+ idempotencyKey: 'skipped',+ payload: {},+ createdAt: new Date(Date.now() - 3000),+ });+ await failures.save({+ ...input,+ idempotencyKey: 'bad',+ payload: { body: 'bad' },+ createdAt: new Date(Date.now() - 2000),+ });+ await failures.save({+ ...input,+ idempotencyKey: 'healthy',+ payload: { body: 'healthy' },+ createdAt: new Date(Date.now() - 1000),+ });+ await failures.save({ ...input, event: 'message.ack', idempotencyKey: 'unsubscribed', payload: {} });+ await failures.save({+ ...input,+ idempotencyKey: 'expired',+ payload: {},+ createdAt: new Date(Date.now() - 25 * 3600000),+ });+ const delivery = {+ redeliver: jest.fn(async (_webhook: Webhook, _session: string, _event: string, key: string) => {+ if (key === 'bad') {+ await recordWebhookDeliveryFailure(failures, logger, { ...input, idempotencyKey: key, attempts: 1 });+ return 'failed';+ }+ await failures.delete({ idempotencyKey: key });+ return 'delivered';+ }),+ };+ const service = new WebhookRedriveService(webhooks, failures, delivery as never, config);+ expect(await service.redrive({ limit: 1 }, [sessionId])).toMatchObject({ failed: 1, remaining: 2 });+ expect(await service.redrive({ limit: 1 }, [sessionId])).toMatchObject({ delivered: 1, remaining: 1 });+ expect(delivery.redeliver.mock.calls.map(call => call[3])).toEqual(['bad', 'healthy']);+ const unsubscribed = await failures.findOneByOrFail({ idempotencyKey: 'unsubscribed' });+ await webhooks.update({ id: input.webhookId }, { events: ['*'] });+ expect(await service.redrive({ ids: [unsubscribed.id] })).toMatchObject({ delivered: 1, remaining: 0 });+ const disabled = new WebhookRedriveService(+ webhooks,+ failures,+ delivery as never,+ new ConfigService({ webhook: { failurePayloadRetentionHours: 0 } }),+ );+ expect(await disabled.redrive({})).toMatchObject({ redriven: 0, remaining: 0 });++ expect(await service.redrive({ sessionId: 'outside' }, [sessionId])).toMatchObject({ redriven: 0 });+ });++ it('converges two connections on one terminal identity', async () => {+ const result = await Promise.all([+ recordWebhookDeliveryFailure(failures, logger, input),+ recordWebhookDeliveryFailure(second.getRepository(WebhookDeliveryFailure), logger, { ...input, attempts: 1 }),+ ]);+ expect(result.sort()).toEqual([false, true]);+ expect(await failures.count()).toBe(1);+ expect((await failures.findOneByOrFail({ idempotencyKey: 'key' })).attempts).toBe(4);+ });++ it('upgrades duplicate terminal records while preserving replay data', async () => {+ await ds.query(`DROP INDEX "${schema}"."UQ_webhook_delivery_failures_terminal"`);+ await failures.save({ ...input, payload: { body: 'retained' } });+ await failures.save({ ...input, attempts: 5 });+ const migration = new DeduplicateTerminalWebhookFailures1787200000000();+ await migration.up(ds.createQueryRunner());+ expect(await failures.count()).toBe(1);+ const row = await failures.findOneOrFail({ where: { idempotencyKey: 'key' }, select: { id: true, payload: true } });+ expect(row.payload).toEqual({ body: 'retained' });+ await expect(failures.save({ ...input })).rejects.toThrow();+ });++ it('retains queued outbox data and never reopens a settled row', async () => {+ const repository = ds.getRepository(WebhookOutboxEvent);+ const outbox = new WebhookOutboxService(repository);+ await outbox.open({ ...input, payload: { body: 'retained' }, deliveryId: 'job' });+ await outbox.markQueued(input.webhookId, 'key', 'job');+ await repository.update({ idempotencyKey: 'key' }, { createdAt: new Date(0) });+ expect(await outbox.pruneSettled(1)).toBe(0);+ expect(await outbox.findStale(new Date(), 1)).toEqual([+ expect.objectContaining({ state: 'queued', payload: { body: 'retained' } }),+ ]);+ await outbox.close(input.webhookId, 'key', 'dispatched');+ await outbox.markQueued(input.webhookId, 'key', 'late');+ expect(await repository.findOneByOrFail({ idempotencyKey: 'key' })).toMatchObject({+ state: 'dispatched',+ payload: null,+ });+ });++ it('publishes verified media atomically and preserves concurrent metadata', async () => {+ const repository = ds.getRepository(Message);+ const bytes = Buffer.from('verified media');+ const storage = {+ putFile: jest.fn().mockResolvedValue(undefined),+ getFile: jest.fn().mockResolvedValue(bytes),+ deleteFile: jest.fn().mockResolvedValue(undefined),+ };+ const archive = new ChatMediaArchiveService(+ repository,+ storage as unknown as StorageService,+ new ConfigService({ chatMedia: { archiveEnabled: true, inlineMode: 'archive' } }),+ );+ const metadata = { media: { mimetype: 'image/png', data: bytes.toString('base64') } };+ const base = {+ sessionId,+ chatId: '[email protected]',+ from: '[email protected]',+ to: '[email protected]',+ body: '',+ type: 'image',+ direction: MessageDirection.INCOMING,+ status: MessageStatus.SENT,+ timestamp: 1,+ metadata,+ };+ const first = await repository.save({ ...base, waMessageId: 'success' });+ expect(await archive.archive(first)).not.toBeNull();+ expect((await repository.findOneByOrFail({ id: first.id })).metadata?.media).toMatchObject({+ archived: true,+ omitted: true,+ });+ const raced = await repository.save({ ...base, waMessageId: 'race' });+ const find = repository.findOne.bind(repository);+ jest.spyOn(repository, 'findOne').mockImplementationOnce(async options => {+ const snapshot = await find(options);+ await repository.update({ id: raced.id }, { metadata: { ...metadata, reactions: ['kept'] } });+ return snapshot;+ });+ expect(await archive.archive(raced)).toBeNull();+ expect(await repository.findOneByOrFail({ id: raced.id })).toMatchObject({+ mediaPath: null,+ metadata: { ...metadata, reactions: ['kept'] },+ });+ });+});diff --git a/src/modules/webhook/webhook-redrive.service.ts b/src/modules/webhook/webhook-redrive.service.tsindex ff1d0ad1c..8ca552179 100644--- a/src/modules/webhook/webhook-redrive.service.ts+++ b/src/modules/webhook/webhook-redrive.service.ts@@ -28,11 +28,11 @@ export interface WebhookRedriveRequest {}export interface WebhookRedriveResult {- /** Rows replayed by this call: delivered plus enqueued. */+ /** Rows delivered by this call. */redriven: number;/** Delivered by a direct POST; their failure rows are gone. */delivered: number;- /** Handed to the queue; the row is removed when the job delivers, and kept when it fails again. */+ /** Reserved for compatibility; operator redrive always uses a direct POST and returns zero. */enqueued: number;/** The replay failed again; the row stays, with its attempt count raised. */failed: number;@@ -109,7 +109,11 @@ export class WebhookRedriveService {this.failureRepository.manager.connection.options.type === 'postgres' ? 'subscribed.event' : 'subscribed.value';const query = this.failureRepository.createQueryBuilder('failure')- .innerJoin(Webhook, 'webhook', 'webhook.id = failure.webhookId AND webhook.sessionId = failure.sessionId')+ .innerJoin(+ Webhook,+ 'webhook',+ 'CAST(webhook.id AS text) = failure.webhookId AND CAST(webhook.sessionId AS text) = failure.sessionId',+ ).where({attempts: MoreThan(0),payload: Not(IsNull()),