Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(integration): redrive only dead letters of the current binding

Yudhi Armyndharisea1d05391c02
diff --git a/src/core/plugins/plugin-activation.ts b/src/core/plugins/plugin-activation.ts
index 5151a24d0..cfb7f093b 100644
--- a/src/core/plugins/plugin-activation.ts
+++ b/src/core/plugins/plugin-activation.ts
@@ -1,3 +1,12 @@
+/**
+ * The all-sessions scope is spelled null, undefined or '*' depending on where it was stored; this maps
+ * every spelling to '*' and leaves a concrete session id as it is, so two scopes compare by key.
+ */
+export const sessionScopeKey = (scope: string | null | undefined): string => (!scope || scope === '*' ? '*' : scope);
+
+/** Whether `scope` is the all-sessions scope, in any of its spellings. */
+export const isAllSessionsScope = (scope: string | null | undefined): boolean => sessionScopeKey(scope) === '*';
+
/**
* Per-session activation gate. A plugin declares whether it is session-scoped (the default); the
* operator then activates it for all sessions (`['*']`) or an explicit set. A global plugin
diff --git a/src/modules/integration/ingress-reconciler.service.spec.ts b/src/modules/integration/ingress-reconciler.service.spec.ts
index 80f438762..e45ec62d1 100644
--- a/src/modules/integration/ingress-reconciler.service.spec.ts
+++ b/src/modules/integration/ingress-reconciler.service.spec.ts
@@ -778,7 +778,7 @@ describe('IngressReconcilerService.sweep', () => {
);
const redrive = new RedriveService(failures, events, { enqueue } as unknown as IngressEnqueueService);
- const res = await redrive.redriveInstance('plug', 'inst', null);
+ const res = await redrive.redriveInstance('plug', 'inst', 'sess-1');
expect(res.redriven).toBe(1);
expect((await stored(id)).dispatchState).toBe('dispatched');
@@ -788,6 +788,36 @@ describe('IngressReconcilerService.sweep', () => {
expect(enqueue).not.toHaveBeenCalled();
});
+ it('redrives, on a real database, only the rows written under the current all-sessions binding', async () => {
+ const dead = (deliveryId: string, sessionId: string | null) =>
+ failures.save(
+ failures.create({
+ direction: 'inbound',
+ pluginId: 'plug',
+ instanceId: 'inst',
+ sessionId,
+ deliveryId,
+ attempts: 0,
+ lastError: 'failed',
+ payload: { route: 'chatwoot', ingress: { headers: {}, query: {}, body: '{}', rawBody: '{}' } },
+ redriven: false,
+ }),
+ );
+ await dead('d-none', null);
+ await dead('d-star', '*');
+ await dead('d-old', 'sess-old');
+ const redrive = new RedriveService(failures, events, { enqueue } as unknown as IngressEnqueueService);
+
+ const res = await redrive.redriveInstance('plug', 'inst', null);
+
+ expect(res).toMatchObject({ redriven: 2, remaining: 0 });
+ expect((enqueue.mock.calls as Array<[{ deliveryId: string }]>).map(call => call[0].deliveryId).sort()).toEqual([
+ 'd-none',
+ 'd-star',
+ ]);
+ expect(await failures.count({ where: { sessionId: 'sess-old', redriven: false } })).toBe(1);
+ });
+
it('skips a row whose instance was disabled after persist, and replays it once re-enabled', async () => {
const id = await insertEvent();
resolveInstance.mockResolvedValue({ enabled: false });
diff --git a/src/modules/integration/redrive.controller.spec.ts b/src/modules/integration/redrive.controller.spec.ts
index f1863fad0..4f0d58f8a 100644
--- a/src/modules/integration/redrive.controller.spec.ts
+++ b/src/modules/integration/redrive.controller.spec.ts
@@ -7,6 +7,7 @@ import { AuditAction } from '../audit/entities/audit-log.entity';
import { AuditService } from '../audit/audit.service';
import { PluginInstanceService } from './plugin-instance.service';
import { RedriveService } from './redrive.service';
+import { IngressEnqueueService } from './ingress-enqueue.service';
describe('RedriveController authz', () => {
it('is ADMIN-gated (re-dispatching DLQ payloads can cause real sends; a VIEWER/OPERATOR key must not)', () => {
@@ -103,11 +104,19 @@ describe('RedriveController session-scope fence', () => {
expect(redrive.redriveInstance).not.toHaveBeenCalled();
});
- it('lets an unrestricted key redrive any instance, passing null as an unrestricted provenance filter', async () => {
+ it('lets an unrestricted key redrive any instance, passing the current binding as the provenance filter', async () => {
const { controller, redrive } = build('sess-2');
await controller.redriveInstance('chatwoot', 'acct1', { allowedSessions: null } as unknown as ApiKey);
+ expect(redrive.redriveInstance).toHaveBeenCalledWith('chatwoot', 'acct1', 'sess-2');
+ });
+
+ it('passes null for an instance bound to all sessions', async () => {
+ const { controller, redrive } = build(null);
+
+ await controller.redriveInstance('chatwoot', 'acct1', { allowedSessions: null } as unknown as ApiKey);
+
expect(redrive.redriveInstance).toHaveBeenCalledWith('chatwoot', 'acct1', null);
});
@@ -129,3 +138,80 @@ describe('RedriveController session-scope fence', () => {
expect(audit.logInfo).not.toHaveBeenCalled();
});
});
+
+// A redrive runs the real service against a stored dead letter. The queue is on, so an accepted replay
+// retires its row at once and the job fails later: what the answer says is all the operator sees.
+describe('RedriveController after an instance was rebound', () => {
+ type Row = { id: string; sessionId: string | null; redriven: boolean };
+ const key = { allowedSessions: null } as unknown as ApiKey;
+ const dead = (id: string, sessionId: string | null): Row => ({ id, sessionId, redriven: false });
+
+ function build(scope: string | null, rows: Row[]) {
+ // The where clause as the service writes it: a session id, IsNull(), or a list of alternatives.
+ const matchesSession = (row: Row, sessionId: unknown): boolean =>
+ typeof sessionId === 'string' ? row.sessionId === sessionId : row.sessionId === null;
+ const matches = (row: Row, where: Record<string, unknown>): boolean =>
+ row.redriven === where.redriven && (!('sessionId' in where) || matchesSession(row, where.sessionId));
+ const repo = {
+ find: jest.fn((opts: { where: Record<string, unknown> | Array<Record<string, unknown>> }) => {
+ const alternatives = Array.isArray(opts.where) ? opts.where : [opts.where];
+ return Promise.resolve(
+ rows
+ .filter(row => alternatives.some(where => matches(row, where)))
+ .map(row => ({
+ ...row,
+ direction: 'inbound',
+ deliveryId: `d-${row.id}`,
+ attempts: 0,
+ payload: { route: 'r', ingress: { headers: {}, query: {}, body: '{}', rawBody: '{}' } },
+ })),
+ );
+ }),
+ update: jest.fn((where: { id: string }, patch: Partial<Row>) => {
+ Object.assign(rows.find(row => row.id === where.id) ?? {}, patch);
+ return Promise.resolve();
+ }),
+ count: jest.fn(() => Promise.resolve(rows.filter(row => !row.redriven).length)),
+ increment: jest.fn(),
+ };
+ const events = { update: jest.fn().mockResolvedValue(undefined) };
+ const enqueue = jest
+ .fn<Promise<{ outcome: string }>, [{ deliveryId: string; sessionId?: string }]>()
+ .mockResolvedValue({
+ outcome: 'queued',
+ });
+ const service = new RedriveService(repo as never, events as never, { enqueue } as unknown as IngressEnqueueService);
+ const instances = {
+ resolve: jest
+ .fn()
+ .mockResolvedValue({ pluginId: 'chatwoot', instanceId: 'acct1', sessionScope: scope, enabled: true }),
+ };
+ const audit = { logInfo: jest.fn() };
+ const controller = new RedriveController(
+ service,
+ instances as unknown as PluginInstanceService,
+ audit as unknown as AuditService,
+ );
+ return { controller, enqueue, rows };
+ }
+
+ it('does not replay or retire a dead letter of an earlier session binding', async () => {
+ const { controller, enqueue, rows } = build('sess-new', [dead('old', 'sess-old'), dead('current', 'sess-new')]);
+
+ const result = await controller.redriveInstance('chatwoot', 'acct1', key);
+
+ expect(enqueue).toHaveBeenCalledTimes(1);
+ expect(enqueue.mock.calls[0][0]).toMatchObject({ sessionId: 'sess-new' });
+ expect(result.redriven).toBe(1);
+ expect(rows.find(row => row.id === 'old')?.redriven).toBe(false);
+ });
+
+ it('does not replay a dead letter of a session binding once the instance serves all sessions', async () => {
+ const { controller, enqueue } = build(null, [dead('old', 'sess-old'), dead('all', null), dead('star', '*')]);
+
+ const result = await controller.redriveInstance('chatwoot', 'acct1', key);
+
+ expect(result.redriven).toBe(2);
+ expect(enqueue.mock.calls.map(call => call[0].deliveryId).sort()).toEqual(['d-all', 'd-star']);
+ });
+});
diff --git a/src/modules/integration/redrive.controller.ts b/src/modules/integration/redrive.controller.ts
index 800f35aea..ea3a99ffc 100644
--- a/src/modules/integration/redrive.controller.ts
+++ b/src/modules/integration/redrive.controller.ts
@@ -59,14 +59,13 @@ export class RedriveController {
`instance ${instanceId} of plugin ${pluginId} is ${inst ? 'disabled' : 'deleted'}; its dead-lettered deliveries cannot be redriven`,
);
}
- // Thread the authorized binding down as an explicit DLQ provenance filter. A scoped key is
- // authorized against the instance's CURRENT sessionScope, but retained DLQ rows still carry the
- // sessionId of whatever binding wrote them; without this filter a rebind sess-old -> sess-current
- // lets a sess-current key replay historical sess-old rows. The filter is derived from the
- // PERSISTED current instance (not the request body), and null means an unrestricted caller may
- // drain every retained row — never undefined, which would silently fail open.
- const sessionIdFilter = scoped ? inst.sessionScope : null;
- const result = await this.redrive.redriveInstance(pluginId, instanceId, sessionIdFilter);
+ // Thread the instance's CURRENT binding down as the DLQ provenance filter, for every key. Retained DLQ
+ // rows still carry the sessionId of whatever binding wrote them; without this filter a rebind
+ // sess-old -> sess-current lets a sess-current key replay historical sess-old rows, and lets an
+ // unrestricted key replay them too, although dispatch refuses a delivery of an earlier binding, so
+ // each would be retired here and dead-lettered again. The binding is read from the PERSISTED instance
+ // (not the request body).
+ const result = await this.redrive.redriveInstance(pluginId, instanceId, inst.sessionScope);
// A redrive can trigger real downstream sends, so every successful batch is audited with its
// outcome counts (no payload content — the DLQ rows themselves hold those).
void this.audit.logInfo(AuditAction.INTEGRATION_INSTANCE_REDRIVEN, {
diff --git a/src/modules/integration/redrive.service.spec.ts b/src/modules/integration/redrive.service.spec.ts
index b8dac40f6..e22a8543c 100644
--- a/src/modules/integration/redrive.service.spec.ts
+++ b/src/modules/integration/redrive.service.spec.ts
@@ -173,63 +173,40 @@ describe('RedriveService', () => {
expect(events.update).not.toHaveBeenCalled(); // nothing was delivered, so the event row stays as-is
});
- it('queries only non-redriven inbound rows for the instance and returns 0 when none are found', async () => {
+ it('queries only non-redriven inbound rows of the current binding and returns 0 when none are found', async () => {
(repo.find as jest.Mock).mockResolvedValue([]);
const svc = makeSvc();
- const res = await svc.redriveInstance('p', 'i', null);
+ const res = await svc.redriveInstance('p', 'i', 'sess-current');
+ // The binding enters BOTH the find() and the count() query: rows of an earlier binding neither consume
+ // the batch nor leak through `remaining`, since dispatch refuses them and a replay would only fail again.
expect(repo.find).toHaveBeenCalledWith({
- where: { pluginId: 'p', instanceId: 'i', direction: 'inbound', redriven: false },
+ where: { pluginId: 'p', instanceId: 'i', direction: 'inbound', redriven: false, sessionId: 'sess-current' },
order: { attempts: 'ASC', createdAt: 'ASC' },
take: 100,
});
+ expect(repo.count).toHaveBeenCalledWith({
+ where: { pluginId: 'p', instanceId: 'i', direction: 'inbound', redriven: false, sessionId: 'sess-current' },
+ });
expect(res.redriven).toBe(0);
expect(ingressEnqueue.enqueue).not.toHaveBeenCalled();
expect(repo.update).not.toHaveBeenCalled();
});
- it('filters the DLQ batch by the session provenance so a rebind cannot replay foreign historical rows', async () => {
- // After an instance is rebound sess-old -> sess-current, a key scoped to sess-current must NOT
- // replay retained rows that still carry sessionId 'sess-old'. The authorized binding is threaded
- // in as a concrete filter that enters BOTH the find() and count() query — foreign rows neither
- // consume the batch nor leak through `remaining`.
+ it.each([null, '*'])('takes a row with no session or "*" for an instance bound to all sessions (%s)', async scope => {
(repo.find as jest.Mock).mockResolvedValue([]);
const svc = makeSvc();
- await svc.redriveInstance('p', 'i', 'sess-current');
+ await svc.redriveInstance('p', 'i', scope);
- expect(repo.find).toHaveBeenCalledWith({
- where: {
- pluginId: 'p',
- instanceId: 'i',
- direction: 'inbound',
- redriven: false,
- sessionId: 'sess-current',
- },
- order: { attempts: 'ASC', createdAt: 'ASC' },
- take: 100,
- });
- const [countCall] = (repo.count as jest.Mock).mock.calls[0] as [{ where: Record<string, unknown> }];
- expect(countCall.where).toMatchObject({ sessionId: 'sess-current' });
- });
-
- it('leaves the sessionId predicate ABSENT for an unrestricted (null) caller so legacy/null-session rows stay replayable', async () => {
- // null means the caller is unrestricted: the DLQ is drained across every sessionId, including
- // legacy rows written before sessionId provenance existed (sessionId null). The where object must
- // NOT carry a sessionId key at all — emitting `sessionId: null` would exclude those legacy rows.
- (repo.find as jest.Mock).mockResolvedValue([]);
- const svc = makeSvc();
-
- await svc.redriveInstance('p', 'i', null);
-
- expect(repo.find).toHaveBeenCalledWith({
- where: { pluginId: 'p', instanceId: 'i', direction: 'inbound', redriven: false },
- order: { attempts: 'ASC', createdAt: 'ASC' },
- take: 100,
- });
- const [countArg] = (repo.count as jest.Mock).mock.calls[0] as [{ where: Record<string, unknown> }];
- expect(countArg.where).not.toHaveProperty('sessionId');
+ const base = { pluginId: 'p', instanceId: 'i', direction: 'inbound', redriven: false };
+ const where = [
+ { ...base, sessionId: expect.objectContaining({ _type: 'isNull' }) as unknown },
+ { ...base, sessionId: '*' },
+ ];
+ expect(repo.find).toHaveBeenCalledWith({ where, order: { attempts: 'ASC', createdAt: 'ASC' }, take: 100 });
+ expect(repo.count).toHaveBeenCalledWith({ where });
});
it('falls back to the failure row id as deliveryId when deliveryId is null', async () => {
diff --git a/src/modules/integration/redrive.service.ts b/src/modules/integration/redrive.service.ts
index 99d6c1aff..8e47ad4cd 100644
--- a/src/modules/integration/redrive.service.ts
+++ b/src/modules/integration/redrive.service.ts
@@ -1,6 +1,6 @@
import { Injectable, OnModuleDestroy } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
-import { Repository } from 'typeorm';
+import { IsNull, Repository } from 'typeorm';
import { IntegrationDeliveryFailure } from './entities/integration-delivery-failure.entity';
import { IngressEvent } from './entities/ingress-event.entity';
import { IngressEnqueueService } from './ingress-enqueue.service';
@@ -8,6 +8,7 @@ import { IngressJobData } from '../queue/processors/ingress.processor';
import { KeyedAsyncLock } from './ordering-lock';
import { INGRESS_DISPATCH_TIMEOUT_MS } from './integration.constants';
import { createLogger } from '../../common/services/logger.service';
+import { isAllSessionsScope } from '../../core/plugins/plugin-activation';
const REDRIVE_BATCH_SIZE = 100;
@@ -59,13 +60,18 @@ export class RedriveService implements OnModuleDestroy {
}
}
+ /**
+ * `sessionScope` is the instance's CURRENT binding (null or '*' for all sessions). Only rows written under
+ * that binding are replayed or counted: dispatch refuses a delivery whose instance was bound to another
+ * session since, so a row of an earlier binding would be retired here and dead-lettered again by its job.
+ */
redriveInstance(
pluginId: string,
instanceId: string,
- sessionIdFilter: string | null,
+ sessionScope: string | null,
): Promise<{ redriven: number; remaining: number; batchSize: number }> {
const run = this.lock.run(`redrive:${pluginId}:${instanceId}`, () =>
- this.redriveBatch(pluginId, instanceId, sessionIdFilter),
+ this.redriveBatch(pluginId, instanceId, sessionScope),
);
this.inFlight.add(run);
const forget = () => void this.inFlight.delete(run);
@@ -76,15 +82,16 @@ export class RedriveService implements OnModuleDestroy {
private async redriveBatch(
pluginId: string,
instanceId: string,
- sessionIdFilter: string | null,
+ sessionScope: string | null,
): Promise<{ redriven: number; remaining: number; batchSize: number }> {
- const where = {
- pluginId,
- instanceId,
- direction: 'inbound' as const,
- redriven: false,
- ...(sessionIdFilter === null ? {} : { sessionId: sessionIdFilter }),
- };
+ const base = { pluginId, instanceId, direction: 'inbound' as const, redriven: false };
+ // A job of the all-sessions scope carries no session or '*', whichever way it was stored.
+ const where = isAllSessionsScope(sessionScope)
+ ? [
+ { ...base, sessionId: IsNull() },
+ { ...base, sessionId: '*' },
+ ]
+ : { ...base, sessionId: sessionScope as string };
// Failed replays increment attempts and move behind never-retried rows, preventing one permanent
// failure from livelocking the bounded window while keeping it redrivable.
const order = { attempts: 'ASC', createdAt: 'ASC' } as const;