Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(webhook): read past a stored payload that does not parse
Yudhi Armyndharise2d2dd84696e
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex e81520002..b10927e70 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -56,6 +56,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- A `CORS_ORIGINS` entry written with a path or a trailing slash is used as the origin it names, and a warning at boot lists it, where it matched no `Origin` header.- Index ingress dead letters by delivery id, so the reconciler's per-delivery checks stop scanning every dead letter of an instance; the first boot builds the index.- Replay stranded ingress events past one whose stored payload is not valid JSON, instead of failing every sweep on it.+- Replay stranded webhook deliveries and redrive ingress dead letters past a row whose stored payload is not valid JSON, instead of failing every pass on it.- Hand a chat back to the bot when the plugin instance that took it over is deleted, instead of silencing every other plugin on it.- Keep the sessions of a plugin's other enabled instances active when creating, disabling or deleting a session-scoped instance retires an operator-set `*`.- Omit the name, push name, caption, background colour and font of a received status that has none from `GET /status` instead of returning `null`, as the schema declares.diff --git a/src/modules/integration/redrive.service.spec.ts b/src/modules/integration/redrive.service.spec.tsindex 85b49661f..29e7412b0 100644--- a/src/modules/integration/redrive.service.spec.ts+++ b/src/modules/integration/redrive.service.spec.ts@@ -11,7 +11,12 @@ describe('RedriveService', () => {let ingressEnqueue: jest.Mocked<Partial<IngressEnqueueService>>;beforeEach(() => {- repo = { find: jest.fn(), update: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(0) };+ repo = {+ find: jest.fn(),+ findOneBy: jest.fn(),+ update: jest.fn().mockResolvedValue(undefined),+ count: jest.fn().mockResolvedValue(0),+ };events = { update: jest.fn().mockResolvedValue(undefined) };ingressEnqueue = { enqueue: jest.fn().mockResolvedValue({ outcome: 'queued' }) };});@@ -24,6 +29,22 @@ describe('RedriveService', () => {);}+ it('redrives the rows it can read when one stored payload does not parse', async () => {+ const ingress = { headers: {}, query: {}, body: '{}', rawBody: '{}' };+ const healthy = { id: 'f2', pluginId: 'p', instanceId: 'i', deliveryId: 'd2', payload: { route: 'r', ingress } };+ // The window read fails as a whole; read again by id, the first row cannot be read and the second can.+ (repo.find as jest.Mock)+ .mockRejectedValueOnce(new SyntaxError('Unexpected token o in JSON at position 1'))+ .mockResolvedValueOnce([{ id: 'f1' }, { id: 'f2' }]);+ (repo.findOneBy as jest.Mock).mockRejectedValueOnce(new SyntaxError('bad json')).mockResolvedValueOnce(healthy);++ const res = await makeSvc().redriveInstance('p', 'i', null);++ expect(res.redriven).toBe(1);+ expect(ingressEnqueue.enqueue).toHaveBeenCalledTimes(1);+ expect(repo.update).toHaveBeenCalledWith({ id: 'f2' }, { redriven: true });+ });+it('re-enqueues non-redriven inbound failures for an instance and marks them redriven', async () => {const rows = [{diff --git a/src/modules/integration/redrive.service.ts b/src/modules/integration/redrive.service.tsindex c069b0579..405114582 100644--- a/src/modules/integration/redrive.service.ts+++ b/src/modules/integration/redrive.service.ts@@ -85,13 +85,37 @@ export class RedriveService implements OnModuleDestroy {redriven: false,...(sessionIdFilter === null ? {} : { sessionId: sessionIdFilter }),};- const rows = await this.repo.find({- where,- // Failed replays increment attempts and move behind never-retried rows, preventing one permanent- // failure from livelocking the bounded window while keeping it redrivable.- order: { attempts: 'ASC', createdAt: 'ASC' },- take: REDRIVE_BATCH_SIZE,- });+ // Failed replays increment attempts and move behind never-retried rows, preventing one permanent+ // failure from livelocking the bounded window while keeping it redrivable.+ const order = { attempts: 'ASC', createdAt: 'ASC' } as const;+ let rows: IntegrationDeliveryFailure[];+ try {+ rows = await this.repo.find({ where, order, take: REDRIVE_BATCH_SIZE });+ } catch (err) {+ // A stored payload that does not parse (an imported or hand-edited dead letter) fails the read of+ // the whole window, and it heads the window for good since it is never replayed, so every redrive+ // of the instance would fail with it. Read the window again row by row and skip the unreadable one.+ this.logger.warn('Reading the dead letters of an instance failed; reading them row by row', {+ pluginId,+ instanceId,+ error: err instanceof Error ? err.message : String(err),+ action: 'ingress_redrive_window_unreadable',+ });+ const ids = await this.repo.find({ select: { id: true }, where, order, take: REDRIVE_BATCH_SIZE });+ rows = [];+ for (const { id } of ids) {+ try {+ const row = await this.repo.findOneBy({ id });+ if (row) rows.push(row);+ } catch (rowError) {+ this.logger.error('A dead letter could not be read and is skipped', {+ id,+ error: rowError instanceof Error ? rowError.message : String(rowError),+ action: 'ingress_redrive_row_unreadable',+ });+ }+ }+ }let redriven = 0;for (const row of rows) {diff --git a/src/modules/webhook/webhook-outbox.service.sqlite.spec.ts b/src/modules/webhook/webhook-outbox.service.sqlite.spec.tsindex b693dda1d..bcb372297 100644--- a/src/modules/webhook/webhook-outbox.service.sqlite.spec.ts+++ b/src/modules/webhook/webhook-outbox.service.sqlite.spec.ts@@ -53,3 +53,43 @@ describe('WebhookOutboxService.open() against a concurrent write (SQLite)', () =expect(lost).toEqual([]);});});++// The payload column is text on SQLite and is parsed when a row is read. A row whose stored text is not+// JSON (an imported or hand-edited row) must cost the sweep that row only, not every row behind it.+describe('WebhookOutboxService.findStale() past an unreadable payload (SQLite)', () => {+ let ds: DataSource;+ let outbox: WebhookOutboxService;++ beforeEach(async () => {+ ds = await new DataSource({+ type: 'better-sqlite3',+ database: ':memory:',+ entities: [Session, Webhook, WebhookDeliveryFailure, WebhookOutboxEvent],+ synchronize: true,+ }).initialize();+ outbox = new WebhookOutboxService(ds.getRepository(WebhookOutboxEvent));+ });+ afterEach(() => ds.destroy());++ const insertRaw = async (id: string, key: string, payload: string, createdAt: string): Promise<void> => {+ await ds.query(+ `INSERT INTO webhook_outbox_events (id, webhookId, sessionId, event, idempotencyKey, deliveryId, payload, state, attempts, createdAt)+ VALUES (?, 'w1', 's1', 'message.received', ?, ?, ?, 'pending', 0, ?)`,+ [id, key, `d-${key}`, payload, createdAt],+ );+ };++ it('returns the readable rows behind it and reports how far it scanned', async () => {+ await insertRaw('00000000-0000-4000-8000-000000000001', 'k1', '{oops', '2026-01-01 00:00:00');+ await insertRaw('00000000-0000-4000-8000-000000000002', 'k2', '{"a":1}', '2026-01-01 00:00:01');+ await insertRaw('00000000-0000-4000-8000-000000000003', 'k3', '{oops', '2026-01-01 00:00:02');++ let scannedThrough: string | undefined;+ const rows = await outbox.findStale(new Date('2026-02-01T00:00:00Z'), 10, undefined, id => (scannedThrough = id));++ expect(rows.map(r => r.idempotencyKey)).toEqual(['k2']);+ expect(rows[0].payload).toEqual({ a: 1 });+ // The cursor passes the unreadable row at the end of the page, or a page of them would be read again forever.+ expect(scannedThrough).toBe('00000000-0000-4000-8000-000000000003');+ });+});diff --git a/src/modules/webhook/webhook-outbox.service.ts b/src/modules/webhook/webhook-outbox.service.tsindex a7c97553b..0fa101b6f 100644--- a/src/modules/webhook/webhook-outbox.service.ts+++ b/src/modules/webhook/webhook-outbox.service.ts@@ -138,27 +138,61 @@ export class WebhookOutboxService implements OnModuleInit, OnModuleDestroy {}}- /** Unsettled rows older than the staleness window; queued jobs are checked before replay. */- async findStale(olderThan: Date, limit: number, afterId?: string): Promise<ReplayableDelivery[]> {+ /**+ * Unsettled rows older than the staleness window; queued jobs are checked before replay.+ *+ * A stored payload that does not parse (an imported or hand-edited row on SQLite, where it is text)+ * fails the read of every page it sits in, so no row behind it would ever be replayed. When the page+ * read fails it is read again row by row, an unreadable row is logged and skipped, and `onScanned`+ * receives the last id the page covered so a caller's cursor passes the unreadable rows too.+ */+ async findStale(+ olderThan: Date,+ limit: number,+ afterId?: string,+ onScanned?: (lastId: string) => void,+ ): Promise<ReplayableDelivery[]> {const pending = { state: In(['pending', 'queued']), payload: Not(IsNull()), createdAt: LessThan(olderThan) };- const rows = await this.outbox.find({- where: afterId- ? {- ...pending,- // Compare in SQL so PostgreSQL's sub-millisecond timestamps are never rounded through Date.- createdAt: And(- LessThan(olderThan),- Raw(- alias =>- `(${alias}, "id") > (SELECT "createdAt", "id" FROM "webhook_outbox_events" WHERE "id" = :afterId)`,- { afterId },- ),+ const where = afterId+ ? {+ ...pending,+ // Compare in SQL so PostgreSQL's sub-millisecond timestamps are never rounded through Date.+ createdAt: And(+ LessThan(olderThan),+ Raw(+ alias =>+ `(${alias}, "id") > (SELECT "createdAt", "id" FROM "webhook_outbox_events" WHERE "id" = :afterId)`,+ { afterId },),- }- : pending,- order: { createdAt: 'ASC', id: 'ASC' },- take: limit,- });+ ),+ }+ : pending;+ const order = { createdAt: 'ASC', id: 'ASC' } as const;+ let rows: WebhookOutboxEvent[];+ try {+ rows = await this.outbox.find({ where, order, take: limit });+ } catch (error) {+ this.logger.warn('Reading a page of stranded deliveries failed; reading it row by row', {+ error: error instanceof Error ? error.message : String(error),+ action: 'webhook_outbox_page_unreadable',+ });+ const ids = await this.outbox.find({ select: { id: true }, where, order, take: limit });+ rows = [];+ for (const { id } of ids) {+ try {+ const row = await this.outbox.findOneBy({ id });+ if (row) rows.push(row);+ } catch (rowError) {+ this.logger.error('A stranded delivery could not be read and is skipped', {+ id,+ error: rowError instanceof Error ? rowError.message : String(rowError),+ action: 'webhook_outbox_row_unreadable',+ });+ }+ }+ const last = ids.at(-1);+ if (last) onScanned?.(last.id);+ }// A pending row always carries its payload; the guard is for a row whose payload was retired by// a concurrent close between the query and this read.return rowsdiff --git a/src/modules/webhook/webhook-reconciler.service.spec.ts b/src/modules/webhook/webhook-reconciler.service.spec.tsindex 0c130c0a4..66c8b3a14 100644--- a/src/modules/webhook/webhook-reconciler.service.spec.ts+++ b/src/modules/webhook/webhook-reconciler.service.spec.ts@@ -233,6 +233,20 @@ describe('WebhookReconcilerService', () => {expect(stats).toMatchObject({ scanned: 1, skipped: 1, replayed: 0, failed: 0 });});+ it('moves its cursor past rows a page read could not return, instead of reading them again', async () => {+ // A page made only of rows whose stored payload is unreadable returns nothing, but it was scanned.+ outbox.findStale.mockImplementationOnce((_cutoff, _limit, _after, onScanned?: (lastId: string) => void) => {+ onScanned?.('row-9');+ return Promise.resolve([]);+ });++ await service.sweep(OPTS);+ expect(outbox.findStale).toHaveBeenCalledTimes(1);++ await service.sweep(OPTS);+ expect((outbox.findStale.mock.calls as unknown[][])[1][2]).toBe('row-9');+ });+it('never stacks a second pass on top of a slow one', async () => {let release: () => void = () => {};outbox.findStale.mockImplementation(diff --git a/src/modules/webhook/webhook-reconciler.service.ts b/src/modules/webhook/webhook-reconciler.service.tsindex 41827c2d7..4b119b103 100644--- a/src/modules/webhook/webhook-reconciler.service.ts+++ b/src/modules/webhook/webhook-reconciler.service.ts@@ -133,13 +133,19 @@ export class WebhookReconcilerService implements OnModuleInit, OnModuleDestroy {this.inFlight = new Promise(resolve => (settle = resolve));try {const cutoff = new Date(now.getTime() - opts.graceMs);- let rows = await this.outbox.findStale(cutoff, opts.batchSize, this.cursor);- if (rows.length === 0 && this.cursor) {+ // The last id the page covered, which can lie beyond its last readable row: a row whose stored+ // payload is unreadable is skipped, and the cursor must pass it or a page of them is read forever.+ let scannedThrough: string | undefined;+ const onScanned = (id: string): void => {+ scannedThrough = id;+ };+ let rows = await this.outbox.findStale(cutoff, opts.batchSize, this.cursor, onScanned);+ if (rows.length === 0 && !scannedThrough && this.cursor) {this.cursor = undefined;- rows = await this.outbox.findStale(cutoff, opts.batchSize);+ rows = await this.outbox.findStale(cutoff, opts.batchSize, undefined, onScanned);}// Live jobs may occupy a whole page. Advance past them without spending their replay budget.- this.cursor = rows.at(-1)?.id;+ this.cursor = scannedThrough ?? rows.at(-1)?.id;stats.scanned = rows.length;for (const row of rows) {if (this.stop.signal.aborted) break;