Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(integration): redrive only dead letters of the current binding
Yudhi Armyndharisea1d05391c02
diff --git a/src/core/plugins/plugin-activation.ts b/src/core/plugins/plugin-activation.tsindex 5151a24d0..cfb7f093b 100644--- a/src/core/plugins/plugin-activation.ts+++ b/src/core/plugins/plugin-activation.ts@@ -1,3 +1,12 @@+/**+ * The all-sessions scope is spelled null, undefined or '*' depending on where it was stored; this maps+ * every spelling to '*' and leaves a concrete session id as it is, so two scopes compare by key.+ */+export const sessionScopeKey = (scope: string | null | undefined): string => (!scope || scope === '*' ? '*' : scope);++/** Whether `scope` is the all-sessions scope, in any of its spellings. */+export const isAllSessionsScope = (scope: string | null | undefined): boolean => sessionScopeKey(scope) === '*';+/*** Per-session activation gate. A plugin declares whether it is session-scoped (the default); the* operator then activates it for all sessions (`['*']`) or an explicit set. A global plugindiff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.tsindex 80f438762..e45ec62d1 100644--- a/src/modules/integration/ingress-reconciler.service.spec.ts+++ b/src/modules/integration/ingress-reconciler.service.spec.ts@@ -778,7 +778,7 @@ describe('IngressReconcilerService.sweep', () => {);const redrive = new RedriveService(failures, events, { enqueue } as unknown as IngressEnqueueService);- const res = await redrive.redriveInstance('plug', 'inst', null);+ const res = await redrive.redriveInstance('plug', 'inst', 'sess-1');expect(res.redriven).toBe(1);expect((await stored(id)).dispatchState).toBe('dispatched');@@ -788,6 +788,36 @@ describe('IngressReconcilerService.sweep', () => {expect(enqueue).not.toHaveBeenCalled();});+ it('redrives, on a real database, only the rows written under the current all-sessions binding', async () => {+ const dead = (deliveryId: string, sessionId: string | null) =>+ failures.save(+ failures.create({+ direction: 'inbound',+ pluginId: 'plug',+ instanceId: 'inst',+ sessionId,+ deliveryId,+ attempts: 0,+ lastError: 'failed',+ payload: { route: 'chatwoot', ingress: { headers: {}, query: {}, body: '{}', rawBody: '{}' } },+ redriven: false,+ }),+ );+ await dead('d-none', null);+ await dead('d-star', '*');+ await dead('d-old', 'sess-old');+ const redrive = new RedriveService(failures, events, { enqueue } as unknown as IngressEnqueueService);++ const res = await redrive.redriveInstance('plug', 'inst', null);++ expect(res).toMatchObject({ redriven: 2, remaining: 0 });+ expect((enqueue.mock.calls as Array<[{ deliveryId: string }]>).map(call => call[0].deliveryId).sort()).toEqual([+ 'd-none',+ 'd-star',+ ]);+ expect(await failures.count({ where: { sessionId: 'sess-old', redriven: false } })).toBe(1);+ });+it('skips a row whose instance was disabled after persist, and replays it once re-enabled', async () => {const id = await insertEvent();resolveInstance.mockResolvedValue({ enabled: false });diff --git a/src/modules/integration/redrive.controller.spec.ts b/src/modules/integration/redrive.controller.spec.tsindex f1863fad0..4f0d58f8a 100644--- a/src/modules/integration/redrive.controller.spec.ts+++ b/src/modules/integration/redrive.controller.spec.ts@@ -7,6 +7,7 @@ import { AuditAction } from '../audit/entities/audit-log.entity';import { AuditService } from '../audit/audit.service';import { PluginInstanceService } from './plugin-instance.service';import { RedriveService } from './redrive.service';+import { IngressEnqueueService } from './ingress-enqueue.service';describe('RedriveController authz', () => {it('is ADMIN-gated (re-dispatching DLQ payloads can cause real sends; a VIEWER/OPERATOR key must not)', () => {@@ -103,11 +104,19 @@ describe('RedriveController session-scope fence', () => {expect(redrive.redriveInstance).not.toHaveBeenCalled();});- it('lets an unrestricted key redrive any instance, passing null as an unrestricted provenance filter', async () => {+ it('lets an unrestricted key redrive any instance, passing the current binding as the provenance filter', async () => {const { controller, redrive } = build('sess-2');await controller.redriveInstance('chatwoot', 'acct1', { allowedSessions: null } as unknown as ApiKey);+ expect(redrive.redriveInstance).toHaveBeenCalledWith('chatwoot', 'acct1', 'sess-2');+ });++ it('passes null for an instance bound to all sessions', async () => {+ const { controller, redrive } = build(null);++ await controller.redriveInstance('chatwoot', 'acct1', { allowedSessions: null } as unknown as ApiKey);+expect(redrive.redriveInstance).toHaveBeenCalledWith('chatwoot', 'acct1', null);});@@ -129,3 +138,80 @@ describe('RedriveController session-scope fence', () => {expect(audit.logInfo).not.toHaveBeenCalled();});});++// A redrive runs the real service against a stored dead letter. The queue is on, so an accepted replay+// retires its row at once and the job fails later: what the answer says is all the operator sees.+describe('RedriveController after an instance was rebound', () => {+ type Row = { id: string; sessionId: string | null; redriven: boolean };+ const key = { allowedSessions: null } as unknown as ApiKey;+ const dead = (id: string, sessionId: string | null): Row => ({ id, sessionId, redriven: false });++ function build(scope: string | null, rows: Row[]) {+ // The where clause as the service writes it: a session id, IsNull(), or a list of alternatives.+ const matchesSession = (row: Row, sessionId: unknown): boolean =>+ typeof sessionId === 'string' ? row.sessionId === sessionId : row.sessionId === null;+ const matches = (row: Row, where: Record<string, unknown>): boolean =>+ row.redriven === where.redriven && (!('sessionId' in where) || matchesSession(row, where.sessionId));+ const repo = {+ find: jest.fn((opts: { where: Record<string, unknown> | Array<Record<string, unknown>> }) => {+ const alternatives = Array.isArray(opts.where) ? opts.where : [opts.where];+ return Promise.resolve(+ rows+ .filter(row => alternatives.some(where => matches(row, where)))+ .map(row => ({+ ...row,+ direction: 'inbound',+ deliveryId: `d-${row.id}`,+ attempts: 0,+ payload: { route: 'r', ingress: { headers: {}, query: {}, body: '{}', rawBody: '{}' } },+ })),+ );+ }),+ update: jest.fn((where: { id: string }, patch: Partial<Row>) => {+ Object.assign(rows.find(row => row.id === where.id) ?? {}, patch);+ return Promise.resolve();+ }),+ count: jest.fn(() => Promise.resolve(rows.filter(row => !row.redriven).length)),+ increment: jest.fn(),+ };+ const events = { update: jest.fn().mockResolvedValue(undefined) };+ const enqueue = jest+ .fn<Promise<{ outcome: string }>, [{ deliveryId: string; sessionId?: string }]>()+ .mockResolvedValue({+ outcome: 'queued',+ });+ const service = new RedriveService(repo as never, events as never, { enqueue } as unknown as IngressEnqueueService);+ const instances = {+ resolve: jest+ .fn()+ .mockResolvedValue({ pluginId: 'chatwoot', instanceId: 'acct1', sessionScope: scope, enabled: true }),+ };+ const audit = { logInfo: jest.fn() };+ const controller = new RedriveController(+ service,+ instances as unknown as PluginInstanceService,+ audit as unknown as AuditService,+ );+ return { controller, enqueue, rows };+ }++ it('does not replay or retire a dead letter of an earlier session binding', async () => {+ const { controller, enqueue, rows } = build('sess-new', [dead('old', 'sess-old'), dead('current', 'sess-new')]);++ const result = await controller.redriveInstance('chatwoot', 'acct1', key);++ expect(enqueue).toHaveBeenCalledTimes(1);+ expect(enqueue.mock.calls[0][0]).toMatchObject({ sessionId: 'sess-new' });+ expect(result.redriven).toBe(1);+ expect(rows.find(row => row.id === 'old')?.redriven).toBe(false);+ });++ it('does not replay a dead letter of a session binding once the instance serves all sessions', async () => {+ const { controller, enqueue } = build(null, [dead('old', 'sess-old'), dead('all', null), dead('star', '*')]);++ const result = await controller.redriveInstance('chatwoot', 'acct1', key);++ expect(result.redriven).toBe(2);+ expect(enqueue.mock.calls.map(call => call[0].deliveryId).sort()).toEqual(['d-all', 'd-star']);+ });+});diff --git a/src/modules/integration/redrive.controller.ts b/src/modules/integration/redrive.controller.tsindex 800f35aea..ea3a99ffc 100644--- a/src/modules/integration/redrive.controller.ts+++ b/src/modules/integration/redrive.controller.ts@@ -59,14 +59,13 @@ export class RedriveController {`instance ${instanceId} of plugin ${pluginId} is ${inst ? 'disabled' : 'deleted'}; its dead-lettered deliveries cannot be redriven`,);}- // Thread the authorized binding down as an explicit DLQ provenance filter. A scoped key is- // authorized against the instance's CURRENT sessionScope, but retained DLQ rows still carry the- // sessionId of whatever binding wrote them; without this filter a rebind sess-old -> sess-current- // lets a sess-current key replay historical sess-old rows. The filter is derived from the- // PERSISTED current instance (not the request body), and null means an unrestricted caller may- // drain every retained row — never undefined, which would silently fail open.- const sessionIdFilter = scoped ? inst.sessionScope : null;- const result = await this.redrive.redriveInstance(pluginId, instanceId, sessionIdFilter);+ // Thread the instance's CURRENT binding down as the DLQ provenance filter, for every key. Retained DLQ+ // rows still carry the sessionId of whatever binding wrote them; without this filter a rebind+ // sess-old -> sess-current lets a sess-current key replay historical sess-old rows, and lets an+ // unrestricted key replay them too, although dispatch refuses a delivery of an earlier binding, so+ // each would be retired here and dead-lettered again. The binding is read from the PERSISTED instance+ // (not the request body).+ const result = await this.redrive.redriveInstance(pluginId, instanceId, inst.sessionScope);// A redrive can trigger real downstream sends, so every successful batch is audited with its// outcome counts (no payload content — the DLQ rows themselves hold those).void this.audit.logInfo(AuditAction.INTEGRATION_INSTANCE_REDRIVEN, {diff --git a/src/modules/integration/redrive.service.spec.ts b/src/modules/integration/redrive.service.spec.tsindex b8dac40f6..e22a8543c 100644--- a/src/modules/integration/redrive.service.spec.ts+++ b/src/modules/integration/redrive.service.spec.ts@@ -173,63 +173,40 @@ describe('RedriveService', () => {expect(events.update).not.toHaveBeenCalled(); // nothing was delivered, so the event row stays as-is});- it('queries only non-redriven inbound rows for the instance and returns 0 when none are found', async () => {+ it('queries only non-redriven inbound rows of the current binding and returns 0 when none are found', async () => {(repo.find as jest.Mock).mockResolvedValue([]);const svc = makeSvc();- const res = await svc.redriveInstance('p', 'i', null);+ const res = await svc.redriveInstance('p', 'i', 'sess-current');+ // The binding enters BOTH the find() and the count() query: rows of an earlier binding neither consume+ // the batch nor leak through `remaining`, since dispatch refuses them and a replay would only fail again.expect(repo.find).toHaveBeenCalledWith({- where: { pluginId: 'p', instanceId: 'i', direction: 'inbound', redriven: false },+ where: { pluginId: 'p', instanceId: 'i', direction: 'inbound', redriven: false, sessionId: 'sess-current' },order: { attempts: 'ASC', createdAt: 'ASC' },take: 100,});+ expect(repo.count).toHaveBeenCalledWith({+ where: { pluginId: 'p', instanceId: 'i', direction: 'inbound', redriven: false, sessionId: 'sess-current' },+ });expect(res.redriven).toBe(0);expect(ingressEnqueue.enqueue).not.toHaveBeenCalled();expect(repo.update).not.toHaveBeenCalled();});- it('filters the DLQ batch by the session provenance so a rebind cannot replay foreign historical rows', async () => {- // After an instance is rebound sess-old -> sess-current, a key scoped to sess-current must NOT- // replay retained rows that still carry sessionId 'sess-old'. The authorized binding is threaded- // in as a concrete filter that enters BOTH the find() and count() query — foreign rows neither- // consume the batch nor leak through `remaining`.+ it.each([null, '*'])('takes a row with no session or "*" for an instance bound to all sessions (%s)', async scope => {(repo.find as jest.Mock).mockResolvedValue([]);const svc = makeSvc();- await svc.redriveInstance('p', 'i', 'sess-current');+ await svc.redriveInstance('p', 'i', scope);- expect(repo.find).toHaveBeenCalledWith({- where: {- pluginId: 'p',- instanceId: 'i',- direction: 'inbound',- redriven: false,- sessionId: 'sess-current',- },- order: { attempts: 'ASC', createdAt: 'ASC' },- take: 100,- });- const [countCall] = (repo.count as jest.Mock).mock.calls[0] as [{ where: Record<string, unknown> }];- expect(countCall.where).toMatchObject({ sessionId: 'sess-current' });- });-- it('leaves the sessionId predicate ABSENT for an unrestricted (null) caller so legacy/null-session rows stay replayable', async () => {- // null means the caller is unrestricted: the DLQ is drained across every sessionId, including- // legacy rows written before sessionId provenance existed (sessionId null). The where object must- // NOT carry a sessionId key at all — emitting `sessionId: null` would exclude those legacy rows.- (repo.find as jest.Mock).mockResolvedValue([]);- const svc = makeSvc();-- await svc.redriveInstance('p', 'i', null);-- expect(repo.find).toHaveBeenCalledWith({- where: { pluginId: 'p', instanceId: 'i', direction: 'inbound', redriven: false },- order: { attempts: 'ASC', createdAt: 'ASC' },- take: 100,- });- const [countArg] = (repo.count as jest.Mock).mock.calls[0] as [{ where: Record<string, unknown> }];- expect(countArg.where).not.toHaveProperty('sessionId');+ const base = { pluginId: 'p', instanceId: 'i', direction: 'inbound', redriven: false };+ const where = [+ { ...base, sessionId: expect.objectContaining({ _type: 'isNull' }) as unknown },+ { ...base, sessionId: '*' },+ ];+ expect(repo.find).toHaveBeenCalledWith({ where, order: { attempts: 'ASC', createdAt: 'ASC' }, take: 100 });+ expect(repo.count).toHaveBeenCalledWith({ where });});it('falls back to the failure row id as deliveryId when deliveryId is null', async () => {diff --git a/src/modules/integration/redrive.service.ts b/src/modules/integration/redrive.service.tsindex 99d6c1aff..8e47ad4cd 100644--- a/src/modules/integration/redrive.service.ts+++ b/src/modules/integration/redrive.service.ts@@ -1,6 +1,6 @@import { Injectable, OnModuleDestroy } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';-import { Repository } from 'typeorm';+import { IsNull, Repository } from 'typeorm';import { IntegrationDeliveryFailure } from './entities/integration-delivery-failure.entity';import { IngressEvent } from './entities/ingress-event.entity';import { IngressEnqueueService } from './ingress-enqueue.service';@@ -8,6 +8,7 @@ import { IngressJobData } from '../queue/processors/ingress.processor';import { KeyedAsyncLock } from './ordering-lock';import { INGRESS_DISPATCH_TIMEOUT_MS } from './integration.constants';import { createLogger } from '../../common/services/logger.service';+import { isAllSessionsScope } from '../../core/plugins/plugin-activation';const REDRIVE_BATCH_SIZE = 100;@@ -59,13 +60,18 @@ export class RedriveService implements OnModuleDestroy {}}+ /**+ * `sessionScope` is the instance's CURRENT binding (null or '*' for all sessions). Only rows written under+ * that binding are replayed or counted: dispatch refuses a delivery whose instance was bound to another+ * session since, so a row of an earlier binding would be retired here and dead-lettered again by its job.+ */redriveInstance(pluginId: string,instanceId: string,- sessionIdFilter: string | null,+ sessionScope: string | null,): Promise<{ redriven: number; remaining: number; batchSize: number }> {const run = this.lock.run(`redrive:${pluginId}:${instanceId}`, () =>- this.redriveBatch(pluginId, instanceId, sessionIdFilter),+ this.redriveBatch(pluginId, instanceId, sessionScope),);this.inFlight.add(run);const forget = () => void this.inFlight.delete(run);@@ -76,15 +82,16 @@ export class RedriveService implements OnModuleDestroy {private async redriveBatch(pluginId: string,instanceId: string,- sessionIdFilter: string | null,+ sessionScope: string | null,): Promise<{ redriven: number; remaining: number; batchSize: number }> {- const where = {- pluginId,- instanceId,- direction: 'inbound' as const,- redriven: false,- ...(sessionIdFilter === null ? {} : { sessionId: sessionIdFilter }),- };+ const base = { pluginId, instanceId, direction: 'inbound' as const, redriven: false };+ // A job of the all-sessions scope carries no session or '*', whichever way it was stored.+ const where = isAllSessionsScope(sessionScope)+ ? [+ { ...base, sessionId: IsNull() },+ { ...base, sessionId: '*' },+ ]+ : { ...base, sessionId: sessionScope as string };// Failed replays increment attempts and move behind never-retried rows, preventing one permanent// failure from livelocking the bounded window while keeping it redrivable.const order = { attempts: 'ASC', createdAt: 'ASC' } as const;