Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(webhook): stop an operator redrive at shutdown (#1840)

Yudhi Armyndharisfb81a30153d0
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 400b0d20c..c66d61f2e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Stop an operator webhook redrive at its next row on shutdown.
- Run a plugin's `onDisable` once when shutdown, uninstall or an update overlaps a disable already in progress.
- Answer `503` and keep the previous version when shutdown begins before an update of a plugin the operator switched on has applied.
- Disable plugins whose enable is still running at shutdown, after the running ones, and refuse new plugin enables during teardown.
diff --git a/docs/06-api-specification.md b/docs/06-api-specification.md
index 41e89b0bf..397655f43 100644
--- a/docs/06-api-specification.md
+++ b/docs/06-api-specification.md
@@ -4745,7 +4745,7 @@ Queued deliveries retain their pre-hook outbox copy until the worker completes d
Replay recorded webhook deliveries that still hold their event data, in one bounded batch, with the lowest attempt counts first, then oldest first. A row is replayable when it is terminal (`attempts` > 0; an `attempts: 0` row is the outbox's to replay) and was recorded while `WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS` > 0, until that window passes. With the default `0` nothing is kept and this call replays nothing.
-Each replay reuses the row's stored `idempotencyKey` (sent as `X-OpenWA-Idempotency-Key`), so receivers can deduplicate a POST that timed out after processing. `webhook:before` hooks run again on the stored pre-hook data. Each row gets **one direct POST** inside the request, including when ordinary dispatch uses the queue. Success removes its row; another failure keeps it and raises its attempt count. Calls on one node run serially. The audit log records counts without payload content.
+Each replay reuses the row's stored `idempotencyKey` (sent as `X-OpenWA-Idempotency-Key`), so receivers can deduplicate a POST that timed out after processing. `webhook:before` hooks run again on the stored pre-hook data. Each row gets **one direct POST** inside the request, including when ordinary dispatch uses the queue. Success removes its row; another failure keeps it and raises its attempt count. Calls on one node run serially. Graceful shutdown stops a batch at its next row and waits for the replays still running (at most four), including their `webhook:before` hooks; each POST is bounded by `WEBHOOK_TIMEOUT`. Rows it did not reach stay for a later call, and a call queued behind it replays nothing. The audit log records counts without payload content.
Choose a batch limit that fits the client's timeout, especially for unavailable receivers. A client
timeout does not cancel the batch; inspect the failure log before retrying it.
diff --git a/src/modules/webhook/webhook-redrive.service.spec.ts b/src/modules/webhook/webhook-redrive.service.spec.ts
index adef532b6..23db973f7 100644
--- a/src/modules/webhook/webhook-redrive.service.spec.ts
+++ b/src/modules/webhook/webhook-redrive.service.spec.ts
@@ -1,6 +1,6 @@
import { ConfigService } from '@nestjs/config';
import { randomUUID } from 'node:crypto';
-import { DataSource, Repository } from 'typeorm';
+import { DataSource, Repository, SelectQueryBuilder } from 'typeorm';
import { Session, SessionStatus } from '../session/entities/session.entity';
import { Webhook } from './entities/webhook.entity';
import { WebhookDeliveryFailure } from './entities/webhook-delivery-failure.entity';
@@ -333,4 +333,35 @@ describe('WebhookRedriveService', () => {
expect(await service.redrive({})).toMatchObject({ delivered: 10, remaining: 0 });
expect(peak).toBe(4);
});
+
+ it('stops a batch at shutdown and waits for the replays in hand', async () => {
+ for (let i = 0; i < 6; i++) await addFailure();
+ const deliver = delivery.redeliver.getMockImplementation()!;
+ let release!: () => void;
+ const gate = new Promise<void>(resolve => (release = resolve));
+ let inHand!: () => void;
+ const fourInHand = new Promise<void>(resolve => (inHand = resolve));
+ delivery.redeliver.mockImplementation(async (...args) => {
+ if (delivery.redeliver.mock.calls.length === 4) inHand();
+ await gate;
+ return deliver(...args);
+ });
+
+ const batch = service.redrive({});
+ await fourInHand;
+ // The batch has read its rows; a call queued behind it must read none.
+ const select = jest.spyOn(SelectQueryBuilder.prototype, 'getMany');
+ const queued = service.redrive({});
+ let destroyed = false;
+ const destroy = service.onModuleDestroy().then(() => (destroyed = true));
+ for (let i = 0; i < 5; i++) await new Promise(resolve => setImmediate(resolve));
+ expect(destroyed).toBe(false);
+
+ release();
+ await destroy;
+ expect(await batch).toMatchObject({ delivered: 4, remaining: 2 });
+ expect(await queued).toMatchObject({ delivered: 0, remaining: 2 });
+ expect(select).not.toHaveBeenCalled();
+ expect(delivery.redeliver).toHaveBeenCalledTimes(4);
+ });
});
diff --git a/src/modules/webhook/webhook-redrive.service.ts b/src/modules/webhook/webhook-redrive.service.ts
index 8ca552179..cb2e42b6e 100644
--- a/src/modules/webhook/webhook-redrive.service.ts
+++ b/src/modules/webhook/webhook-redrive.service.ts
@@ -1,4 +1,4 @@
-import { Injectable } from '@nestjs/common';
+import { Injectable, OnModuleDestroy } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { InjectRepository } from '@nestjs/typeorm';
import { In, IsNull, MoreThan, Not, Repository } from 'typeorm';
@@ -55,7 +55,7 @@ export interface WebhookRedriveResult {
* replay only raises its attempt count.
*/
@Injectable()
-export class WebhookRedriveService {
+export class WebhookRedriveService implements OnModuleDestroy {
private readonly logger = createLogger('WebhookRedrive');
/**
* One redrive at a time on this node, so two overlapping calls (a double click, a retrying script)
@@ -63,6 +63,8 @@ export class WebhookRedriveService {
* idempotency key, which the receiver dedups on.
*/
private running: Promise<unknown> = Promise.resolve();
+ // Aborted on destroy: a batch in progress takes no further row.
+ private readonly stop = new AbortController();
constructor(
@InjectRepository(Webhook, 'data')
@@ -73,6 +75,19 @@ export class WebhookRedriveService {
private readonly configService: ConfigService,
) {}
+ /**
+ * A redrive runs inside an admitted HTTP request, which shutdown lets finish, and its replays hold
+ * no dispatch slot, so the delivery drain neither waits for nor stops them. Left alone it would go
+ * on POSTing after PluginLoaderService has unregistered the `webhook:before` hooks, sending the
+ * stored pre-hook data. Stop at the next row and wait for the replays in hand: this module is
+ * destroyed before the global plugin module, so those still run with their hooks. Rows not reached
+ * stay for a later call and are counted in `remaining`.
+ */
+ async onModuleDestroy(): Promise<void> {
+ this.stop.abort();
+ await this.running;
+ }
+
redrive(request: WebhookRedriveRequest, allowedSessions?: string[] | null): Promise<WebhookRedriveResult> {
const run = this.running.then(() => this.redriveBatch(request, allowedSessions));
// The chain must survive a failed batch, or every later call would inherit its rejection.
@@ -126,16 +141,19 @@ export class WebhookRedriveService {
.andWhere('webhook.active = :active', { active: true })
.andWhere(`EXISTS (SELECT 1 FROM ${subscriptions} WHERE ${subscribedEvent} IN ('*', failure.event))`);
const limit = Math.min(Math.max(1, request.limit ?? DEFAULT_WEBHOOK_REDRIVE_LIMIT), MAX_WEBHOOK_REDRIVE_LIMIT);
- const rows = await query
- .clone()
- // `payload` is select: false on the entity; replay explicitly reads it.
- .addSelect('failure.payload')
- // Failed replays move behind rows with fewer attempts, so a bad receiver cannot pin a batch.
- .orderBy('failure.attempts', 'ASC')
- .addOrderBy('failure.createdAt', 'ASC')
- .addOrderBy('failure.id', 'ASC')
- .take(limit)
- .getMany();
+ // A call still queued when shutdown starts reads no rows (or their payloads); it only counts them.
+ const rows = this.stop.signal.aborted
+ ? []
+ : await query
+ .clone()
+ // `payload` is select: false on the entity; replay explicitly reads it.
+ .addSelect('failure.payload')
+ // Failed replays move behind rows with fewer attempts, so a bad receiver cannot pin a batch.
+ .orderBy('failure.attempts', 'ASC')
+ .addOrderBy('failure.createdAt', 'ASC')
+ .addOrderBy('failure.id', 'ASC')
+ .take(limit)
+ .getMany();
if (rows.length > 0) {
const webhookIds = [...new Set(rows.map(r => r.webhookId))];
@@ -144,7 +162,7 @@ export class WebhookRedriveService {
);
const queue = [...rows];
const worker = async (): Promise<void> => {
- for (let row = queue.shift(); row; row = queue.shift()) {
+ for (let row = queue.shift(); row && !this.stop.signal.aborted; row = queue.shift()) {
await this.redriveRow(row, webhooks.get(row.webhookId) ?? null, result);
}
};