Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(webhook): read past a stored payload that does not parse

Yudhi Armyndharise2d2dd84696e
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e81520002..b10927e70 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -56,6 +56,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- A `CORS_ORIGINS` entry written with a path or a trailing slash is used as the origin it names, and a warning at boot lists it, where it matched no `Origin` header.
- Index ingress dead letters by delivery id, so the reconciler's per-delivery checks stop scanning every dead letter of an instance; the first boot builds the index.
- Replay stranded ingress events past one whose stored payload is not valid JSON, instead of failing every sweep on it.
+- Replay stranded webhook deliveries and redrive ingress dead letters past a row whose stored payload is not valid JSON, instead of failing every pass on it.
- Hand a chat back to the bot when the plugin instance that took it over is deleted, instead of silencing every other plugin on it.
- Keep the sessions of a plugin's other enabled instances active when creating, disabling or deleting a session-scoped instance retires an operator-set `*`.
- Omit the name, push name, caption, background colour and font of a received status that has none from `GET /status` instead of returning `null`, as the schema declares.
diff --git a/src/modules/integration/redrive.service.spec.ts b/src/modules/integration/redrive.service.spec.ts
index 85b49661f..29e7412b0 100644
--- a/src/modules/integration/redrive.service.spec.ts
+++ b/src/modules/integration/redrive.service.spec.ts
@@ -11,7 +11,12 @@ describe('RedriveService', () => {
let ingressEnqueue: jest.Mocked<Partial<IngressEnqueueService>>;
beforeEach(() => {
- repo = { find: jest.fn(), update: jest.fn().mockResolvedValue(undefined), count: jest.fn().mockResolvedValue(0) };
+ repo = {
+ find: jest.fn(),
+ findOneBy: jest.fn(),
+ update: jest.fn().mockResolvedValue(undefined),
+ count: jest.fn().mockResolvedValue(0),
+ };
events = { update: jest.fn().mockResolvedValue(undefined) };
ingressEnqueue = { enqueue: jest.fn().mockResolvedValue({ outcome: 'queued' }) };
});
@@ -24,6 +29,22 @@ describe('RedriveService', () => {
);
}
+ it('redrives the rows it can read when one stored payload does not parse', async () => {
+ const ingress = { headers: {}, query: {}, body: '{}', rawBody: '{}' };
+ const healthy = { id: 'f2', pluginId: 'p', instanceId: 'i', deliveryId: 'd2', payload: { route: 'r', ingress } };
+ // The window read fails as a whole; read again by id, the first row cannot be read and the second can.
+ (repo.find as jest.Mock)
+ .mockRejectedValueOnce(new SyntaxError('Unexpected token o in JSON at position 1'))
+ .mockResolvedValueOnce([{ id: 'f1' }, { id: 'f2' }]);
+ (repo.findOneBy as jest.Mock).mockRejectedValueOnce(new SyntaxError('bad json')).mockResolvedValueOnce(healthy);
+
+ const res = await makeSvc().redriveInstance('p', 'i', null);
+
+ expect(res.redriven).toBe(1);
+ expect(ingressEnqueue.enqueue).toHaveBeenCalledTimes(1);
+ expect(repo.update).toHaveBeenCalledWith({ id: 'f2' }, { redriven: true });
+ });
+
it('re-enqueues non-redriven inbound failures for an instance and marks them redriven', async () => {
const rows = [
{
diff --git a/src/modules/integration/redrive.service.ts b/src/modules/integration/redrive.service.ts
index c069b0579..405114582 100644
--- a/src/modules/integration/redrive.service.ts
+++ b/src/modules/integration/redrive.service.ts
@@ -85,13 +85,37 @@ export class RedriveService implements OnModuleDestroy {
redriven: false,
...(sessionIdFilter === null ? {} : { sessionId: sessionIdFilter }),
};
- const rows = await this.repo.find({
- where,
- // Failed replays increment attempts and move behind never-retried rows, preventing one permanent
- // failure from livelocking the bounded window while keeping it redrivable.
- order: { attempts: 'ASC', createdAt: 'ASC' },
- take: REDRIVE_BATCH_SIZE,
- });
+ // Failed replays increment attempts and move behind never-retried rows, preventing one permanent
+ // failure from livelocking the bounded window while keeping it redrivable.
+ const order = { attempts: 'ASC', createdAt: 'ASC' } as const;
+ let rows: IntegrationDeliveryFailure[];
+ try {
+ rows = await this.repo.find({ where, order, take: REDRIVE_BATCH_SIZE });
+ } catch (err) {
+ // A stored payload that does not parse (an imported or hand-edited dead letter) fails the read of
+ // the whole window, and it heads the window for good since it is never replayed, so every redrive
+ // of the instance would fail with it. Read the window again row by row and skip the unreadable one.
+ this.logger.warn('Reading the dead letters of an instance failed; reading them row by row', {
+ pluginId,
+ instanceId,
+ error: err instanceof Error ? err.message : String(err),
+ action: 'ingress_redrive_window_unreadable',
+ });
+ const ids = await this.repo.find({ select: { id: true }, where, order, take: REDRIVE_BATCH_SIZE });
+ rows = [];
+ for (const { id } of ids) {
+ try {
+ const row = await this.repo.findOneBy({ id });
+ if (row) rows.push(row);
+ } catch (rowError) {
+ this.logger.error('A dead letter could not be read and is skipped', {
+ id,
+ error: rowError instanceof Error ? rowError.message : String(rowError),
+ action: 'ingress_redrive_row_unreadable',
+ });
+ }
+ }
+ }
let redriven = 0;
for (const row of rows) {
diff --git a/src/modules/webhook/webhook-outbox.service.sqlite.spec.ts b/src/modules/webhook/webhook-outbox.service.sqlite.spec.ts
index b693dda1d..bcb372297 100644
--- a/src/modules/webhook/webhook-outbox.service.sqlite.spec.ts
+++ b/src/modules/webhook/webhook-outbox.service.sqlite.spec.ts
@@ -53,3 +53,43 @@ describe('WebhookOutboxService.open() against a concurrent write (SQLite)', () =
expect(lost).toEqual([]);
});
});
+
+// The payload column is text on SQLite and is parsed when a row is read. A row whose stored text is not
+// JSON (an imported or hand-edited row) must cost the sweep that row only, not every row behind it.
+describe('WebhookOutboxService.findStale() past an unreadable payload (SQLite)', () => {
+ let ds: DataSource;
+ let outbox: WebhookOutboxService;
+
+ beforeEach(async () => {
+ ds = await new DataSource({
+ type: 'better-sqlite3',
+ database: ':memory:',
+ entities: [Session, Webhook, WebhookDeliveryFailure, WebhookOutboxEvent],
+ synchronize: true,
+ }).initialize();
+ outbox = new WebhookOutboxService(ds.getRepository(WebhookOutboxEvent));
+ });
+ afterEach(() => ds.destroy());
+
+ const insertRaw = async (id: string, key: string, payload: string, createdAt: string): Promise<void> => {
+ await ds.query(
+ `INSERT INTO webhook_outbox_events (id, webhookId, sessionId, event, idempotencyKey, deliveryId, payload, state, attempts, createdAt)
+ VALUES (?, 'w1', 's1', 'message.received', ?, ?, ?, 'pending', 0, ?)`,
+ [id, key, `d-${key}`, payload, createdAt],
+ );
+ };
+
+ it('returns the readable rows behind it and reports how far it scanned', async () => {
+ await insertRaw('00000000-0000-4000-8000-000000000001', 'k1', '{oops', '2026-01-01 00:00:00');
+ await insertRaw('00000000-0000-4000-8000-000000000002', 'k2', '{"a":1}', '2026-01-01 00:00:01');
+ await insertRaw('00000000-0000-4000-8000-000000000003', 'k3', '{oops', '2026-01-01 00:00:02');
+
+ let scannedThrough: string | undefined;
+ const rows = await outbox.findStale(new Date('2026-02-01T00:00:00Z'), 10, undefined, id => (scannedThrough = id));
+
+ expect(rows.map(r => r.idempotencyKey)).toEqual(['k2']);
+ expect(rows[0].payload).toEqual({ a: 1 });
+ // The cursor passes the unreadable row at the end of the page, or a page of them would be read again forever.
+ expect(scannedThrough).toBe('00000000-0000-4000-8000-000000000003');
+ });
+});
diff --git a/src/modules/webhook/webhook-outbox.service.ts b/src/modules/webhook/webhook-outbox.service.ts
index a7c97553b..0fa101b6f 100644
--- a/src/modules/webhook/webhook-outbox.service.ts
+++ b/src/modules/webhook/webhook-outbox.service.ts
@@ -138,27 +138,61 @@ export class WebhookOutboxService implements OnModuleInit, OnModuleDestroy {
}
}
- /** Unsettled rows older than the staleness window; queued jobs are checked before replay. */
- async findStale(olderThan: Date, limit: number, afterId?: string): Promise<ReplayableDelivery[]> {
+ /**
+ * Unsettled rows older than the staleness window; queued jobs are checked before replay.
+ *
+ * A stored payload that does not parse (an imported or hand-edited row on SQLite, where it is text)
+ * fails the read of every page it sits in, so no row behind it would ever be replayed. When the page
+ * read fails it is read again row by row, an unreadable row is logged and skipped, and `onScanned`
+ * receives the last id the page covered so a caller's cursor passes the unreadable rows too.
+ */
+ async findStale(
+ olderThan: Date,
+ limit: number,
+ afterId?: string,
+ onScanned?: (lastId: string) => void,
+ ): Promise<ReplayableDelivery[]> {
const pending = { state: In(['pending', 'queued']), payload: Not(IsNull()), createdAt: LessThan(olderThan) };
- const rows = await this.outbox.find({
- where: afterId
- ? {
- ...pending,
- // Compare in SQL so PostgreSQL's sub-millisecond timestamps are never rounded through Date.
- createdAt: And(
- LessThan(olderThan),
- Raw(
- alias =>
- `(${alias}, "id") > (SELECT "createdAt", "id" FROM "webhook_outbox_events" WHERE "id" = :afterId)`,
- { afterId },
- ),
+ const where = afterId
+ ? {
+ ...pending,
+ // Compare in SQL so PostgreSQL's sub-millisecond timestamps are never rounded through Date.
+ createdAt: And(
+ LessThan(olderThan),
+ Raw(
+ alias =>
+ `(${alias}, "id") > (SELECT "createdAt", "id" FROM "webhook_outbox_events" WHERE "id" = :afterId)`,
+ { afterId },
),
- }
- : pending,
- order: { createdAt: 'ASC', id: 'ASC' },
- take: limit,
- });
+ ),
+ }
+ : pending;
+ const order = { createdAt: 'ASC', id: 'ASC' } as const;
+ let rows: WebhookOutboxEvent[];
+ try {
+ rows = await this.outbox.find({ where, order, take: limit });
+ } catch (error) {
+ this.logger.warn('Reading a page of stranded deliveries failed; reading it row by row', {
+ error: error instanceof Error ? error.message : String(error),
+ action: 'webhook_outbox_page_unreadable',
+ });
+ const ids = await this.outbox.find({ select: { id: true }, where, order, take: limit });
+ rows = [];
+ for (const { id } of ids) {
+ try {
+ const row = await this.outbox.findOneBy({ id });
+ if (row) rows.push(row);
+ } catch (rowError) {
+ this.logger.error('A stranded delivery could not be read and is skipped', {
+ id,
+ error: rowError instanceof Error ? rowError.message : String(rowError),
+ action: 'webhook_outbox_row_unreadable',
+ });
+ }
+ }
+ const last = ids.at(-1);
+ if (last) onScanned?.(last.id);
+ }
// A pending row always carries its payload; the guard is for a row whose payload was retired by
// a concurrent close between the query and this read.
return rows
diff --git a/src/modules/webhook/webhook-reconciler.service.spec.ts b/src/modules/webhook/webhook-reconciler.service.spec.ts
index 0c130c0a4..66c8b3a14 100644
--- a/src/modules/webhook/webhook-reconciler.service.spec.ts
+++ b/src/modules/webhook/webhook-reconciler.service.spec.ts
@@ -233,6 +233,20 @@ describe('WebhookReconcilerService', () => {
expect(stats).toMatchObject({ scanned: 1, skipped: 1, replayed: 0, failed: 0 });
});
+ it('moves its cursor past rows a page read could not return, instead of reading them again', async () => {
+ // A page made only of rows whose stored payload is unreadable returns nothing, but it was scanned.
+ outbox.findStale.mockImplementationOnce((_cutoff, _limit, _after, onScanned?: (lastId: string) => void) => {
+ onScanned?.('row-9');
+ return Promise.resolve([]);
+ });
+
+ await service.sweep(OPTS);
+ expect(outbox.findStale).toHaveBeenCalledTimes(1);
+
+ await service.sweep(OPTS);
+ expect((outbox.findStale.mock.calls as unknown[][])[1][2]).toBe('row-9');
+ });
+
it('never stacks a second pass on top of a slow one', async () => {
let release: () => void = () => {};
outbox.findStale.mockImplementation(
diff --git a/src/modules/webhook/webhook-reconciler.service.ts b/src/modules/webhook/webhook-reconciler.service.ts
index 41827c2d7..4b119b103 100644
--- a/src/modules/webhook/webhook-reconciler.service.ts
+++ b/src/modules/webhook/webhook-reconciler.service.ts
@@ -133,13 +133,19 @@ export class WebhookReconcilerService implements OnModuleInit, OnModuleDestroy {
this.inFlight = new Promise(resolve => (settle = resolve));
try {
const cutoff = new Date(now.getTime() - opts.graceMs);
- let rows = await this.outbox.findStale(cutoff, opts.batchSize, this.cursor);
- if (rows.length === 0 && this.cursor) {
+ // The last id the page covered, which can lie beyond its last readable row: a row whose stored
+ // payload is unreadable is skipped, and the cursor must pass it or a page of them is read forever.
+ let scannedThrough: string | undefined;
+ const onScanned = (id: string): void => {
+ scannedThrough = id;
+ };
+ let rows = await this.outbox.findStale(cutoff, opts.batchSize, this.cursor, onScanned);
+ if (rows.length === 0 && !scannedThrough && this.cursor) {
this.cursor = undefined;
- rows = await this.outbox.findStale(cutoff, opts.batchSize);
+ rows = await this.outbox.findStale(cutoff, opts.batchSize, undefined, onScanned);
}
// Live jobs may occupy a whole page. Advance past them without spending their replay budget.
- this.cursor = rows.at(-1)?.id;
+ this.cursor = scannedThrough ?? rows.at(-1)?.id;
stats.scanned = rows.length;
for (const row of rows) {
if (this.stop.signal.aborted) break;