Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(webhook): bound bursts of opt-in events
Yudhi Armyndharisa02a03ee55f2
diff --git a/src/modules/webhook/webhook-delivery.service.spec.ts b/src/modules/webhook/webhook-delivery.service.spec.tsindex c40c4070b..0818029a6 100644--- a/src/modules/webhook/webhook-delivery.service.spec.ts+++ b/src/modules/webhook/webhook-delivery.service.spec.ts@@ -2822,6 +2822,116 @@ describe('WebhookDeliveryService', () => {hanging.forEach(release => release());await a;});++ // An opt-in event fires once per recipient and status, so one flush of buffered receipts can be+ // thousands of dispatches in a single tick. They must not spend the budget every session shares.+ describe('a burst of opt-in events', () => {+ const receipt = (i: number) => ({ id: 'OUT1', participant: `${i}@c.us`, status: 'read' });+ const hangingReceipts: Array<() => void> = [];+ const warnSpy = () =>+ jest.spyOn((service as unknown as { logger: { warn: (...args: unknown[]) => void } }).logger, 'warn');+ const burst = (size: number) =>+ Array.from({ length: size }, (_, i) => service.dispatch('sess-r', 'message.receipt', receipt(i)));+ // Answers the receipts' receiver until every dispatch has settled, parked ones included.+ const drain = async (dispatches: Array<Promise<void>>): Promise<void> => {+ let settled = false;+ void Promise.all(dispatches).then(() => (settled = true));+ while (!settled) {+ hangingReceipts.splice(0).forEach(release => release());+ await new Promise(resolve => setImmediate(resolve));+ }+ };++ beforeEach(() => {+ hangingReceipts.length = 0;+ // 4 slots and a park bound of 8, so the opt-in allowance, a quarter of 8, is 2 events at a time.+ internals().dispatchLimiter = new ConcurrencyLimiter(4, 8);+ internals().dispatchMaxQueued = 8;+ hooksBySession['sess-r'] = [+ createMockWebhook({ id: 'wh-r', url: 'https://receipt.example/hook', events: ['message.receipt'] }),+ ];+ hooksBySession['sess-o'] = [+ createMockWebhook({ id: 'wh-o', url: 'https://other.example/hook', events: ['message.received'] }),+ ];+ mockFetch.mockImplementation((url: string) =>+ url.startsWith('https://receipt.')+ ? new Promise(resolve => hangingReceipts.push(() => resolve({ ok: true, status: 200 })))+ : Promise.resolve({ ok: true, status: 200 }),+ );+ });++ it('leaves the shared dispatch budget to the other sessions and records no failure for the excess', async () => {+ const receipts = burst(14);+ await settle(() => hangingReceipts.length > 1);++ await service.dispatch('sess-o', 'message.received', {});++ // The other session's event went out while the receipts were still waiting on their receiver.+ expect(urlsCalled().filter(url => url.startsWith('https://other.'))).toHaveLength(1);+ expect(insertedFailures).toEqual([]);+ expect(hookManager.execute).not.toHaveBeenCalledWith('webhook:error', expect.anything(), expect.anything());+ // Only the admitted receipts and the other session's event ever opened an outbox row.+ expect(outboxService.open).toHaveBeenCalledTimes(3);++ await drain(receipts);+ expect(outboxService.close).toHaveBeenCalledTimes(3);+ });++ it('warns when it starts dropping, and again with the count once the admitted events are done', async () => {+ const warn = warnSpy();+ const receipts = burst(14);+ await settle(() => hangingReceipts.length > 1);++ expect(warn).toHaveBeenCalledTimes(1);+ expect(warn).toHaveBeenCalledWith(+ expect.any(String),+ expect.objectContaining({ action: 'webhook_opt_in_dropping' }),+ );++ await drain(receipts);++ expect(warn).toHaveBeenCalledTimes(2);+ expect(warn).toHaveBeenLastCalledWith(+ expect.any(String),+ expect.objectContaining({ action: 'webhook_opt_in_dropped', dropped: 12 }),+ );+ });++ it('admits opt-in events again once the admitted ones are done, and warns again on the next burst', async () => {+ const warn = warnSpy();+ for (let round = 0; round < 2; round++) {+ const receipts = burst(5);+ await settle(() => hangingReceipts.length > 1);+ await drain(receipts);+ }++ expect(urlsCalled()).toHaveLength(4);+ expect(warn.mock.calls.filter(([, meta]) => (meta as { dropped?: number }).dropped === 3)).toHaveLength(2);+ });++ it('gives its place back when a dispatch throws', async () => {+ const original = (configService.get as jest.Mock).getMockImplementation() as (key: string) => unknown;+ (configService.get as jest.Mock).mockImplementation((key: string, def?: unknown) => {+ if (key === 'webhook.mediaInlineMaxBytes') throw new Error('config unavailable');+ return original(key) ?? def;+ });+ await Promise.allSettled(burst(2));+ (configService.get as jest.Mock).mockImplementation(original);++ const receipts = burst(2);+ await settle(() => hangingReceipts.length > 1);++ expect(hangingReceipts).toHaveLength(2);+ await drain(receipts);+ });++ it('never holds back an event that is not opt-in', async () => {+ await Promise.all(Array.from({ length: 6 }, () => service.dispatch('sess-o', 'message.received', {})));++ expect(urlsCalled()).toHaveLength(6);+ expect(insertedFailures).toEqual([]);+ });+ });});describe('delivery-failure dead-letter', () => {diff --git a/src/modules/webhook/webhook-delivery.service.ts b/src/modules/webhook/webhook-delivery.service.tsindex f126bee82..aa3009e9c 100644--- a/src/modules/webhook/webhook-delivery.service.ts+++ b/src/modules/webhook/webhook-delivery.service.ts@@ -194,6 +194,14 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {* The promise is cached, not its result, so a burst of receipts on a cold entry shares one lookup.*/private readonly optInSubscriptions = new Map<string, { events: Promise<ReadonlySet<string>>; expiresAt: number }>();+ /**+ * Opt-in events admitted by dispatch() and not yet settled. One flush of buffered receipts is+ * thousands of dispatches in a tick, so they hold at most a quarter of the park budget, as a session+ * gate does; the excess is dropped before it opens an outbox row, a failure row or a hook call.+ */+ private optInPending = 0;+ /** Opt-in events dropped since the allowance last emptied, reported when it does. */+ private optInDropped = 0;constructor(@InjectRepository(Webhook, 'data')@@ -285,9 +293,34 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {}async dispatch(sessionId: string, event: string, data: Record<string, unknown>): Promise<void> {+ if (!WILDCARD_EXCLUDED_EVENTS.includes(event)) return this.fanOut(sessionId, event, data);// An opt-in event fires per recipient (about 2,000 per message to a 1,000-member group): skip it// before the per-dispatch webhook read when no active webhook of the session names it.- if (WILDCARD_EXCLUDED_EVENTS.includes(event) && !(await this.sessionNamesOptInEvent(sessionId, event))) return;+ if (!(await this.sessionNamesOptInEvent(sessionId, event))) return;+ if (this.optInPending >= Math.max(1, Math.floor(this.dispatchMaxQueued / 4))) {+ if (this.optInDropped++ === 0) {+ this.logger.warn(+ `Dropping ${event} events: more are pending than a quarter of WEBHOOK_DISPATCH_MAX_QUEUED allows`,+ { event, pending: this.optInPending, action: 'webhook_opt_in_dropping' },+ );+ }+ return;+ }+ this.optInPending++;+ try {+ await this.fanOut(sessionId, event, data);+ } finally {+ if (--this.optInPending === 0 && this.optInDropped > 0) {+ this.logger.warn(`Dropped ${this.optInDropped} opt-in events while their allowance was full`, {+ dropped: this.optInDropped,+ action: 'webhook_opt_in_dropped',+ });+ this.optInDropped = 0;+ }+ }+ }++ private async fanOut(sessionId: string, event: string, data: Record<string, unknown>): Promise<void> {const webhooks = await this.loadActiveWebhooks(sessionId, event);const matchingWebhooks = this.filterMatchingWebhooks(webhooks, event, data);