Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(webhook): bound bursts of opt-in events

Yudhi Armyndharisa02a03ee55f2
diff --git a/src/modules/webhook/webhook-delivery.service.spec.ts b/src/modules/webhook/webhook-delivery.service.spec.ts
index c40c4070b..0818029a6 100644
--- a/src/modules/webhook/webhook-delivery.service.spec.ts
+++ b/src/modules/webhook/webhook-delivery.service.spec.ts
@@ -2822,6 +2822,116 @@ describe('WebhookDeliveryService', () => {
hanging.forEach(release => release());
await a;
});
+
+ // An opt-in event fires once per recipient and status, so one flush of buffered receipts can be
+ // thousands of dispatches in a single tick. They must not spend the budget every session shares.
+ describe('a burst of opt-in events', () => {
+ const receipt = (i: number) => ({ id: 'OUT1', participant: `${i}@c.us`, status: 'read' });
+ const hangingReceipts: Array<() => void> = [];
+ const warnSpy = () =>
+ jest.spyOn((service as unknown as { logger: { warn: (...args: unknown[]) => void } }).logger, 'warn');
+ const burst = (size: number) =>
+ Array.from({ length: size }, (_, i) => service.dispatch('sess-r', 'message.receipt', receipt(i)));
+ // Answers the receipts' receiver until every dispatch has settled, parked ones included.
+ const drain = async (dispatches: Array<Promise<void>>): Promise<void> => {
+ let settled = false;
+ void Promise.all(dispatches).then(() => (settled = true));
+ while (!settled) {
+ hangingReceipts.splice(0).forEach(release => release());
+ await new Promise(resolve => setImmediate(resolve));
+ }
+ };
+
+ beforeEach(() => {
+ hangingReceipts.length = 0;
+ // 4 slots and a park bound of 8, so the opt-in allowance, a quarter of 8, is 2 events at a time.
+ internals().dispatchLimiter = new ConcurrencyLimiter(4, 8);
+ internals().dispatchMaxQueued = 8;
+ hooksBySession['sess-r'] = [
+ createMockWebhook({ id: 'wh-r', url: 'https://receipt.example/hook', events: ['message.receipt'] }),
+ ];
+ hooksBySession['sess-o'] = [
+ createMockWebhook({ id: 'wh-o', url: 'https://other.example/hook', events: ['message.received'] }),
+ ];
+ mockFetch.mockImplementation((url: string) =>
+ url.startsWith('https://receipt.')
+ ? new Promise(resolve => hangingReceipts.push(() => resolve({ ok: true, status: 200 })))
+ : Promise.resolve({ ok: true, status: 200 }),
+ );
+ });
+
+ it('leaves the shared dispatch budget to the other sessions and records no failure for the excess', async () => {
+ const receipts = burst(14);
+ await settle(() => hangingReceipts.length > 1);
+
+ await service.dispatch('sess-o', 'message.received', {});
+
+ // The other session's event went out while the receipts were still waiting on their receiver.
+ expect(urlsCalled().filter(url => url.startsWith('https://other.'))).toHaveLength(1);
+ expect(insertedFailures).toEqual([]);
+ expect(hookManager.execute).not.toHaveBeenCalledWith('webhook:error', expect.anything(), expect.anything());
+ // Only the admitted receipts and the other session's event ever opened an outbox row.
+ expect(outboxService.open).toHaveBeenCalledTimes(3);
+
+ await drain(receipts);
+ expect(outboxService.close).toHaveBeenCalledTimes(3);
+ });
+
+ it('warns when it starts dropping, and again with the count once the admitted events are done', async () => {
+ const warn = warnSpy();
+ const receipts = burst(14);
+ await settle(() => hangingReceipts.length > 1);
+
+ expect(warn).toHaveBeenCalledTimes(1);
+ expect(warn).toHaveBeenCalledWith(
+ expect.any(String),
+ expect.objectContaining({ action: 'webhook_opt_in_dropping' }),
+ );
+
+ await drain(receipts);
+
+ expect(warn).toHaveBeenCalledTimes(2);
+ expect(warn).toHaveBeenLastCalledWith(
+ expect.any(String),
+ expect.objectContaining({ action: 'webhook_opt_in_dropped', dropped: 12 }),
+ );
+ });
+
+ it('admits opt-in events again once the admitted ones are done, and warns again on the next burst', async () => {
+ const warn = warnSpy();
+ for (let round = 0; round < 2; round++) {
+ const receipts = burst(5);
+ await settle(() => hangingReceipts.length > 1);
+ await drain(receipts);
+ }
+
+ expect(urlsCalled()).toHaveLength(4);
+ expect(warn.mock.calls.filter(([, meta]) => (meta as { dropped?: number }).dropped === 3)).toHaveLength(2);
+ });
+
+ it('gives its place back when a dispatch throws', async () => {
+ const original = (configService.get as jest.Mock).getMockImplementation() as (key: string) => unknown;
+ (configService.get as jest.Mock).mockImplementation((key: string, def?: unknown) => {
+ if (key === 'webhook.mediaInlineMaxBytes') throw new Error('config unavailable');
+ return original(key) ?? def;
+ });
+ await Promise.allSettled(burst(2));
+ (configService.get as jest.Mock).mockImplementation(original);
+
+ const receipts = burst(2);
+ await settle(() => hangingReceipts.length > 1);
+
+ expect(hangingReceipts).toHaveLength(2);
+ await drain(receipts);
+ });
+
+ it('never holds back an event that is not opt-in', async () => {
+ await Promise.all(Array.from({ length: 6 }, () => service.dispatch('sess-o', 'message.received', {})));
+
+ expect(urlsCalled()).toHaveLength(6);
+ expect(insertedFailures).toEqual([]);
+ });
+ });
});
describe('delivery-failure dead-letter', () => {
diff --git a/src/modules/webhook/webhook-delivery.service.ts b/src/modules/webhook/webhook-delivery.service.ts
index f126bee82..aa3009e9c 100644
--- a/src/modules/webhook/webhook-delivery.service.ts
+++ b/src/modules/webhook/webhook-delivery.service.ts
@@ -194,6 +194,14 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
* The promise is cached, not its result, so a burst of receipts on a cold entry shares one lookup.
*/
private readonly optInSubscriptions = new Map<string, { events: Promise<ReadonlySet<string>>; expiresAt: number }>();
+ /**
+ * Opt-in events admitted by dispatch() and not yet settled. One flush of buffered receipts is
+ * thousands of dispatches in a tick, so they hold at most a quarter of the park budget, as a session
+ * gate does; the excess is dropped before it opens an outbox row, a failure row or a hook call.
+ */
+ private optInPending = 0;
+ /** Opt-in events dropped since the allowance last emptied, reported when it does. */
+ private optInDropped = 0;
constructor(
@InjectRepository(Webhook, 'data')
@@ -285,9 +293,34 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
}
async dispatch(sessionId: string, event: string, data: Record<string, unknown>): Promise<void> {
+ if (!WILDCARD_EXCLUDED_EVENTS.includes(event)) return this.fanOut(sessionId, event, data);
// An opt-in event fires per recipient (about 2,000 per message to a 1,000-member group): skip it
// before the per-dispatch webhook read when no active webhook of the session names it.
- if (WILDCARD_EXCLUDED_EVENTS.includes(event) && !(await this.sessionNamesOptInEvent(sessionId, event))) return;
+ if (!(await this.sessionNamesOptInEvent(sessionId, event))) return;
+ if (this.optInPending >= Math.max(1, Math.floor(this.dispatchMaxQueued / 4))) {
+ if (this.optInDropped++ === 0) {
+ this.logger.warn(
+ `Dropping ${event} events: more are pending than a quarter of WEBHOOK_DISPATCH_MAX_QUEUED allows`,
+ { event, pending: this.optInPending, action: 'webhook_opt_in_dropping' },
+ );
+ }
+ return;
+ }
+ this.optInPending++;
+ try {
+ await this.fanOut(sessionId, event, data);
+ } finally {
+ if (--this.optInPending === 0 && this.optInDropped > 0) {
+ this.logger.warn(`Dropped ${this.optInDropped} opt-in events while their allowance was full`, {
+ dropped: this.optInDropped,
+ action: 'webhook_opt_in_dropped',
+ });
+ this.optInDropped = 0;
+ }
+ }
+ }
+
+ private async fanOut(sessionId: string, event: string, data: Record<string, unknown>): Promise<void> {
const webhooks = await this.loadActiveWebhooks(sessionId, event);
const matchingWebhooks = this.filterMatchingWebhooks(webhooks, event, data);