Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(webhook): bound opt-in events per session

Yudhi Armyndharisc58ecfdd5d31
diff --git a/src/common/metrics/webhook-delivery-metrics.ts b/src/common/metrics/webhook-delivery-metrics.ts
index e547d2ca8..8f6afc3e1 100644
--- a/src/common/metrics/webhook-delivery-metrics.ts
+++ b/src/common/metrics/webhook-delivery-metrics.ts
@@ -3,7 +3,9 @@
* exhausted, deliveries never sent (shed, refused at shutdown, rejected before sending), and direct
* deliveries stopped by shutdown between retries after earlier attempts were sent. It is
* incremented once per row inserted into the durable `webhook_delivery_failures` dead-letter table;
- * a later replay may replace or remove that row, and the counter keeps the failure. Kept as a plain
+ * a later replay may replace or remove that row, and the counter keeps the failure. It is also
+ * incremented once for each opt-in event dropped past the bound of its session's queue, which files
+ * no row because there is nothing to replay. Kept as a plain
* in-process counter rather than a `COUNT(*)` over that table because the table is pruned on a
* retention schedule, which would make its count non-monotonic and therefore invalid as a
* Prometheus `counter` (a prune would look like a counter reset to `rate()`/`increase()`). An
diff --git a/src/modules/metrics/metrics.service.ts b/src/modules/metrics/metrics.service.ts
index 292be7820..8ed39f400 100644
--- a/src/modules/metrics/metrics.service.ts
+++ b/src/modules/metrics/metrics.service.ts
@@ -221,7 +221,7 @@ export class MetricsService implements OnModuleDestroy {
}
lines.push(
- '# HELP openwa_webhook_delivery_failures_total Webhook delivery failures recorded since process start: retries exhausted, never sent (shed, refused at shutdown, rejected before sending), or stopped by shutdown between direct retries.',
+ '# HELP openwa_webhook_delivery_failures_total Webhook delivery failures recorded since process start: retries exhausted, never sent (shed, refused at shutdown, rejected before sending, or an opt-in event dropped past its session bound), or stopped by shutdown between direct retries.',
);
lines.push('# TYPE openwa_webhook_delivery_failures_total counter');
lines.push(`openwa_webhook_delivery_failures_total ${getWebhookDeliveryFailuresTotal()}`);
diff --git a/src/modules/webhook/webhook-delivery.service.spec.ts b/src/modules/webhook/webhook-delivery.service.spec.ts
index 0818029a6..ddf085be8 100644
--- a/src/modules/webhook/webhook-delivery.service.spec.ts
+++ b/src/modules/webhook/webhook-delivery.service.spec.ts
@@ -1780,6 +1780,38 @@ describe('WebhookDeliveryService', () => {
expect(webhookQueue.add).toHaveBeenCalledWith('webhook-wh-named', expect.anything(), expect.anything());
});
+ // One flush of receipts from a session, with the default dispatch settings: 500 are all enqueued, and of 2000
+ // the 4 that run and the 750 that wait are, the others being dropped and counted.
+ it.each([
+ [500, 500],
+ [2000, 754],
+ ])(
+ 'enqueues %i receipts of one session in a tick as %i jobs, and counts the rest as dropped',
+ async (size, queued) => {
+ const queueService = await buildQueueService((key: string, def?: unknown) =>
+ key === 'queue.enabled' ? true : def,
+ );
+ (repository.find as jest.Mock).mockResolvedValue([createMockWebhook({ events: ['message.receipt'] })]);
+ (hookManager.execute as jest.Mock).mockResolvedValue({ continue: true, data: {} });
+ const dropsBefore = getWebhookDeliveryFailuresTotal();
+
+ await Promise.all(
+ Array.from({ length: size }, (_, i) =>
+ queueService.dispatch('sess-1', 'message.receipt', {
+ id: 'OUT1',
+ participant: `${i}@c.us`,
+ status: 'read',
+ }),
+ ),
+ );
+
+ expect(webhookQueue.add).toHaveBeenCalledTimes(queued);
+ expect(outboxService.open).toHaveBeenCalledTimes(queued);
+ expect(insertedFailures).toEqual([]);
+ expect(getWebhookDeliveryFailuresTotal() - dropsBefore).toBe(size - queued);
+ },
+ );
+
it('uses one direct POST for operator redrive even with the queue enabled', async () => {
const queueService = await buildQueueService((key: string, def?: unknown) =>
key === 'queue.enabled' ? true : def,
@@ -2823,15 +2855,19 @@ describe('WebhookDeliveryService', () => {
await a;
});
- // An opt-in event fires once per recipient and status, so one flush of buffered receipts can be
- // thousands of dispatches in a single tick. They must not spend the budget every session shares.
+ // An opt-in event fires once per recipient and status, so one flush of buffered receipts can be thousands of
+ // dispatches in a tick. Each session passes a stage of its own before the shared pool: a quarter of the
+ // dispatch concurrency of its events run at once, three quarters of WEBHOOK_DISPATCH_MAX_QUEUED more wait their
+ // turn, and the rest are dropped. The pool therefore never holds more than a session's running ones, which keeps
+ // a flood or a dead receiver of one session from shedding or holding back what another session sends.
describe('a burst of opt-in events', () => {
+ type StageInternals = { optInStages: Map<string, unknown> };
const receipt = (i: number) => ({ id: 'OUT1', participant: `${i}@c.us`, status: 'read' });
const hangingReceipts: Array<() => void> = [];
const warnSpy = () =>
jest.spyOn((service as unknown as { logger: { warn: (...args: unknown[]) => void } }).logger, 'warn');
- const burst = (size: number) =>
- Array.from({ length: size }, (_, i) => service.dispatch('sess-r', 'message.receipt', receipt(i)));
+ const burst = (size: number, sessionId = 'sess-r') =>
+ Array.from({ length: size }, (_, i) => service.dispatch(sessionId, 'message.receipt', receipt(i)));
// Answers the receipts' receiver until every dispatch has settled, parked ones included.
const drain = async (dispatches: Array<Promise<void>>): Promise<void> => {
let settled = false;
@@ -2844,48 +2880,65 @@ describe('WebhookDeliveryService', () => {
beforeEach(() => {
hangingReceipts.length = 0;
- // 4 slots and a park bound of 8, so the opt-in allowance, a quarter of 8, is 2 events at a time.
- internals().dispatchLimiter = new ConcurrencyLimiter(4, 8);
+ // 8 slots and a park bound of 8: a session's receipts run 2 at a time (a quarter of 8) and 6 more wait
+ // (three quarters of 8), so 8 of a burst are admitted.
+ internals().dispatchLimiter = new ConcurrencyLimiter(8, 8);
internals().dispatchMaxQueued = 8;
hooksBySession['sess-r'] = [
createMockWebhook({ id: 'wh-r', url: 'https://receipt.example/hook', events: ['message.receipt'] }),
];
+ hooksBySession['sess-q'] = [
+ createMockWebhook({ id: 'wh-q', url: 'https://quiet.example/hook', events: ['message.receipt'] }),
+ ];
+ hooksBySession['sess-f'] = [
+ createMockWebhook({
+ id: 'wh-f',
+ url: 'https://failing.example/hook',
+ events: ['message.receipt'],
+ retryCount: 2,
+ }),
+ ];
hooksBySession['sess-o'] = [
createMockWebhook({ id: 'wh-o', url: 'https://other.example/hook', events: ['message.received'] }),
];
- mockFetch.mockImplementation((url: string) =>
- url.startsWith('https://receipt.')
- ? new Promise(resolve => hangingReceipts.push(() => resolve({ ok: true, status: 200 })))
- : Promise.resolve({ ok: true, status: 200 }),
- );
+ mockFetch.mockImplementation((url: string) => {
+ if (url.startsWith('https://receipt.')) {
+ return new Promise(resolve => hangingReceipts.push(() => resolve({ ok: true, status: 200 })));
+ }
+ if (url.startsWith('https://failing.')) return Promise.reject(new Error('ECONNREFUSED'));
+ return Promise.resolve({ ok: true, status: 200 });
+ });
});
- it('leaves the shared dispatch budget to the other sessions and records no failure for the excess', async () => {
+ it('runs a quarter of the dispatch concurrency at once, queues three quarters of the park bound behind it, and drops the rest', async () => {
+ const dropsBefore = getWebhookDeliveryFailuresTotal();
const receipts = burst(14);
- await settle(() => hangingReceipts.length > 1);
+ await settle(() => hangingReceipts.length > 2);
- await service.dispatch('sess-o', 'message.received', {});
+ // Only the running ones have read the webhooks and opened an outbox row; the queued ones cost nothing yet.
+ expect(hangingReceipts).toHaveLength(2);
+ expect(outboxService.open).toHaveBeenCalledTimes(2);
- // The other session's event went out while the receipts were still waiting on their receiver.
- expect(urlsCalled().filter(url => url.startsWith('https://other.'))).toHaveLength(1);
+ await drain(receipts);
+
+ expect(urlsCalled()).toHaveLength(8);
+ expect(outboxService.open).toHaveBeenCalledTimes(8);
+ expect(outboxService.close).toHaveBeenCalledTimes(8);
+ // A dropped event leaves no failure row, outbox row or hook call: it is only counted, and logged.
expect(insertedFailures).toEqual([]);
expect(hookManager.execute).not.toHaveBeenCalledWith('webhook:error', expect.anything(), expect.anything());
- // Only the admitted receipts and the other session's event ever opened an outbox row.
- expect(outboxService.open).toHaveBeenCalledTimes(3);
-
- await drain(receipts);
- expect(outboxService.close).toHaveBeenCalledTimes(3);
+ expect(getWebhookDeliveryFailuresTotal() - dropsBefore).toBe(6);
});
- it('warns when it starts dropping, and again with the count once the admitted events are done', async () => {
+ it('warns when a session starts dropping, and again with the count once its events are done', async () => {
const warn = warnSpy();
const receipts = burst(14);
- await settle(() => hangingReceipts.length > 1);
+ await settle(() => hangingReceipts.length > 2);
expect(warn).toHaveBeenCalledTimes(1);
expect(warn).toHaveBeenCalledWith(
expect.any(String),
- expect.objectContaining({ action: 'webhook_opt_in_dropping' }),
+ expect.objectContaining({ action: 'webhook_opt_in_dropping', sessionId: 'sess-r' }),
);
await drain(receipts);
@@ -2893,20 +2946,96 @@ describe('WebhookDeliveryService', () => {
expect(warn).toHaveBeenCalledTimes(2);
expect(warn).toHaveBeenLastCalledWith(
expect.any(String),
- expect.objectContaining({ action: 'webhook_opt_in_dropped', dropped: 12 }),
+ expect.objectContaining({ action: 'webhook_opt_in_dropped', sessionId: 'sess-r', dropped: 6 }),
);
});
- it('admits opt-in events again once the admitted ones are done, and warns again on the next burst', async () => {
+ it('drops and logs nothing for a burst its stage holds', async () => {
+ const warn = warnSpy();
+ const dropsBefore = getWebhookDeliveryFailuresTotal();
+
+ await drain(burst(8));
+
+ expect(urlsCalled()).toHaveLength(8);
+ expect(warn).not.toHaveBeenCalled();
+ expect(getWebhookDeliveryFailuresTotal()).toBe(dropsBefore);
+ });
+
+ it("leaves another session's ordinary events and receipts alone while one session holds its stage", async () => {
+ const receipts = burst(14);
+ await settle(() => hangingReceipts.length > 2);
+
+ // The receiver of sess-r never answers, and its stage is full.
+ await service.dispatch('sess-o', 'message.received', {});
+ await service.dispatch('sess-q', 'message.receipt', receipt(1));
+
+ expect(urlsCalled()).toEqual(
+ expect.arrayContaining(['https://other.example/hook', 'https://quiet.example/hook']),
+ );
+ expect(insertedFailures).toEqual([]);
+ expect(hookManager.execute).not.toHaveBeenCalledWith('webhook:error', expect.anything(), expect.anything());
+ await drain(receipts);
+ });
+
+ it("delivers another session's receipt while one session holds its stage with a receiver that keeps failing", async () => {
+ const failing = burst(14, 'sess-f');
+ let failingSettled = false;
+ void Promise.all(failing).then(() => (failingSettled = true));
+ await settle(() => urlsCalled().length > 0);
+
+ await service.dispatch('sess-q', 'message.receipt', receipt(1));
+
+ // It went out while the failing session was still working through its retries.
+ expect(urlsCalled()).toContain('https://quiet.example/hook');
+ expect(failingSettled).toBe(false);
+ await Promise.all(failing);
+ });
+
+ it('forgets a session once its events have settled, and counts the next burst afresh', async () => {
const warn = warnSpy();
for (let round = 0; round < 2; round++) {
- const receipts = burst(5);
- await settle(() => hangingReceipts.length > 1);
+ const receipts = burst(10);
+ await settle(() => hangingReceipts.length > 2);
await drain(receipts);
+ expect((service as unknown as StageInternals).optInStages.size).toBe(0);
}
- expect(urlsCalled()).toHaveLength(4);
- expect(warn.mock.calls.filter(([, meta]) => (meta as { dropped?: number }).dropped === 3)).toHaveLength(2);
+ expect(urlsCalled()).toHaveLength(16);
+ expect(warn.mock.calls.filter(([, meta]) => (meta as { dropped?: number }).dropped === 2)).toHaveLength(2);
+ });
+
+ it('keeps at least one place to run and one to wait however small the pool', async () => {
+ internals().dispatchLimiter = new ConcurrencyLimiter(1, 0);
+ internals().dispatchMaxQueued = 0;
+ const dropsBefore = getWebhookDeliveryFailuresTotal();
+
+ await drain(burst(3));
+
+ expect(urlsCalled()).toHaveLength(2);
+ expect(getWebhookDeliveryFailuresTotal() - dropsBefore).toBe(1);
+ });
+
+ it('drops the events still waiting at shutdown without counting or recording them', async () => {
+ (configService.get as jest.Mock).mockImplementation(<T>(key: string, def?: T): T | boolean | number =>
+ key === 'webhook.shutdownDrainMs' ? 100 : key === 'queue.enabled' ? false : (def as T),
+ );
+ const warn = warnSpy();
+ const dropsBefore = getWebhookDeliveryFailuresTotal();
+ const receipts = burst(14);
+ await settle(() => hangingReceipts.length > 2);
+
+ await service.onModuleDestroy();
+ await drain(receipts);
+
+ // The two that were running finish; the six that waited are gone, and only the six over the bound were
+ // counted and reported.
+ expect(urlsCalled()).toHaveLength(2);
+ expect(getWebhookDeliveryFailuresTotal() - dropsBefore).toBe(6);
+ expect(insertedFailures).toEqual([]);
+ expect(warn).toHaveBeenLastCalledWith(
+ expect.any(String),
+ expect.objectContaining({ action: 'webhook_opt_in_dropped', dropped: 6 }),
+ );
});
it('gives its place back when a dispatch throws', async () => {
@@ -2915,7 +3044,7 @@ describe('WebhookDeliveryService', () => {
if (key === 'webhook.mediaInlineMaxBytes') throw new Error('config unavailable');
return original(key) ?? def;
});
- await Promise.allSettled(burst(2));
+ await Promise.allSettled(burst(8));
(configService.get as jest.Mock).mockImplementation(original);
const receipts = burst(2);
@@ -2931,6 +3060,37 @@ describe('WebhookDeliveryService', () => {
expect(urlsCalled()).toHaveLength(6);
expect(insertedFailures).toEqual([]);
});
+
+ // The sizes an operator gets by default: a flush of 500 receipts is delivered in full, and a flush of 2000
+ // delivers the 4 that run and the 750 that wait.
+ describe('with the default dispatch settings', () => {
+ beforeEach(() => {
+ internals().dispatchLimiter = new ConcurrencyLimiter(16, 1000);
+ internals().dispatchMaxQueued = 1000;
+ mockFetch.mockResolvedValue({ ok: true, status: 200 });
+ });
+
+ it('delivers a flush of 500 receipts from one session in full', async () => {
+ const dropsBefore = getWebhookDeliveryFailuresTotal();
+
+ await Promise.all(burst(500));
+
+ expect(urlsCalled()).toHaveLength(500);
+ expect(insertedFailures).toEqual([]);
+ expect(getWebhookDeliveryFailuresTotal()).toBe(dropsBefore);
+ });
+
+ it('delivers at least 750 of a flush of 2000 and counts every other one as dropped', async () => {
+ const dropsBefore = getWebhookDeliveryFailuresTotal();
+
+ await Promise.all(burst(2000));
+
+ expect(urlsCalled()).toHaveLength(754);
+ expect(outboxService.open).toHaveBeenCalledTimes(754);
+ expect(insertedFailures).toEqual([]);
+ expect(getWebhookDeliveryFailuresTotal() - dropsBefore).toBe(2000 - 754);
+ });
+ });
});
});
diff --git a/src/modules/webhook/webhook-delivery.service.ts b/src/modules/webhook/webhook-delivery.service.ts
index aa3009e9c..5b5530730 100644
--- a/src/modules/webhook/webhook-delivery.service.ts
+++ b/src/modules/webhook/webhook-delivery.service.ts
@@ -135,6 +135,10 @@ class UnrecordedWebhookFailure extends Error {}
const isLimiterClosed = (error: unknown): boolean =>
error instanceof Error && error.message === 'ConcurrencyLimiter closed';
+/** The limiter refused the task because it already runs and queues as many as it admits. */
+const isLimiterFull = (error: unknown): boolean =>
+ error instanceof Error && error.message === 'ConcurrencyLimiter queue full';
+
/**
* The replay's retries were stopped by its aborted signal, in a backoff or just before a retry POST.
* A POST that fails after the abort is not this: on the last attempt it is a terminal failure and is
@@ -195,13 +199,15 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
*/
private readonly optInSubscriptions = new Map<string, { events: Promise<ReadonlySet<string>>; expiresAt: number }>();
/**
- * Opt-in events admitted by dispatch() and not yet settled. One flush of buffered receipts is
- * thousands of dispatches in a tick, so they hold at most a quarter of the park budget, as a session
- * gate does; the excess is dropped before it opens an outbox row, a failure row or a hook call.
+ * Per session, the stage its opt-in events (WILDCARD_EXCLUDED_EVENTS) pass before the shared dispatch
+ * pool. One flush of buffered receipts is thousands of dispatches in a tick: a quarter of the dispatch
+ * concurrency of them run at once, up to three quarters of WEBHOOK_DISPATCH_MAX_QUEUED wait their turn
+ * (holding only the event), and the rest are dropped before they open an outbox row, a failure row or a
+ * hook call. The pool never sees more than the running ones, so a flood, or a receiver that never
+ * answers, cannot shed or hold back what another session sends. An entry lives while its session has
+ * events in the stage; `dropped` counts those turned away since it was created.
*/
- private optInPending = 0;
- /** Opt-in events dropped since the allowance last emptied, reported when it does. */
- private optInDropped = 0;
+ private readonly optInStages = new Map<string, { stage: ConcurrencyLimiter; dropped: number }>();
constructor(
@InjectRepository(Webhook, 'data')
@@ -265,6 +271,9 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
* so the bound limits this hook only; the whole teardown is the sum in shutdown-budget.ts.
*/
async onModuleDestroy(): Promise<void> {
+ // Opt-in events still waiting for their stage hold nothing durable (no outbox row yet) and are best effort,
+ // so they are let go in both modes; the ones already running finish with the rest.
+ for (const { stage } of this.optInStages.values()) stage.close();
if (!this.queueEnabled) {
this.dispatchLimiter.close();
for (const gate of this.degradedSessionGates.values()) gate.close();
@@ -297,29 +306,56 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
// An opt-in event fires per recipient (about 2,000 per message to a 1,000-member group): skip it
// before the per-dispatch webhook read when no active webhook of the session names it.
if (!(await this.sessionNamesOptInEvent(sessionId, event))) return;
- if (this.optInPending >= Math.max(1, Math.floor(this.dispatchMaxQueued / 4))) {
- if (this.optInDropped++ === 0) {
- this.logger.warn(
- `Dropping ${event} events: more are pending than a quarter of WEBHOOK_DISPATCH_MAX_QUEUED allows`,
- { event, pending: this.optInPending, action: 'webhook_opt_in_dropping' },
- );
- }
- return;
+ let entry = this.optInStages.get(sessionId);
+ if (!entry) {
+ // A quarter of the pool runs at once (the limiter keeps at least one) and three quarters of the park bound
+ // wait, at least one place.
+ entry = {
+ stage: new ConcurrencyLimiter(
+ Math.floor(this.dispatchLimiter.maxCount / 4),
+ Math.max(1, Math.floor((this.dispatchMaxQueued * 3) / 4)),
+ ),
+ dropped: 0,
+ };
+ this.optInStages.set(sessionId, entry);
}
- this.optInPending++;
try {
- await this.fanOut(sessionId, event, data);
+ await entry.stage.run(() => this.fanOut(sessionId, event, data));
+ } catch (error) {
+ // Over the stage's bound, or still waiting when shutdown closed it; anything else is not ours to hide.
+ if (isLimiterFull(error)) this.noteOptInDrop(sessionId, event, entry);
+ else if (!isLimiterClosed(error)) throw error;
} finally {
- if (--this.optInPending === 0 && this.optInDropped > 0) {
- this.logger.warn(`Dropped ${this.optInDropped} opt-in events while their allowance was full`, {
- dropped: this.optInDropped,
- action: 'webhook_opt_in_dropped',
- });
- this.optInDropped = 0;
- }
+ this.releaseOptInStage(sessionId, entry);
}
}
+ /**
+ * An opt-in event past its session's bound: counted with the other deliveries that were never sent, and
+ * announced once per burst, when the session starts dropping. It leaves no row to replay, by design.
+ */
+ private noteOptInDrop(sessionId: string, event: string, entry: { dropped: number }): void {
+ incrementWebhookDeliveryFailures();
+ if (entry.dropped++ > 0) return;
+ this.logger.warn(`Dropping ${event} events of session ${sessionId}: its queue of opt-in events is full`, {
+ sessionId,
+ event,
+ action: 'webhook_opt_in_dropping',
+ });
+ }
+
+ /** Forget a session's stage once it is idle, reporting how many events it turned away. */
+ private releaseOptInStage(sessionId: string, entry: { stage: ConcurrencyLimiter; dropped: number }): void {
+ if (entry.stage.activeCount > 0 || entry.stage.queuedCount > 0 || this.optInStages.get(sessionId) !== entry) return;
+ this.optInStages.delete(sessionId);
+ if (entry.dropped === 0) return;
+ this.logger.warn(`Dropped ${entry.dropped} opt-in events of session ${sessionId} while its queue was full`, {
+ sessionId,
+ dropped: entry.dropped,
+ action: 'webhook_opt_in_dropped',
+ });
+ }
+
private async fanOut(sessionId: string, event: string, data: Record<string, unknown>): Promise<void> {
const webhooks = await this.loadActiveWebhooks(sessionId, event);
const matchingWebhooks = this.filterMatchingWebhooks(webhooks, event, data);