Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(webhook): cap a session's receipt deliveries across webhooks
Yudhi Armyndharisbdc0c05e86b1
diff --git a/src/modules/webhook/webhook-delivery.service.spec.ts b/src/modules/webhook/webhook-delivery.service.spec.tsindex 8e3193a5c..df09afdba 100644--- a/src/modules/webhook/webhook-delivery.service.spec.ts+++ b/src/modules/webhook/webhook-delivery.service.spec.ts@@ -2864,7 +2864,7 @@ describe('WebhookDeliveryService', () => {// turn, and the rest are dropped. The pool therefore never holds more than a session's running ones, which keeps// a flood or a dead receiver of one session from shedding or holding back what another session sends.describe('a burst of opt-in events', () => {- type StageInternals = { optInStages: Map<string, unknown> };+ type StageInternals = { optInStages: Map<string, unknown>; optInSessionStages: Map<string, unknown> };const receipt = (i: number) => ({ id: 'OUT1', participant: `${i}@c.us`, status: 'read' });const hangingReceipts: Array<() => void> = [];const warnSpy = () =>@@ -3044,6 +3044,78 @@ describe('WebhookDeliveryService', () => {expect(urlsCalled().filter(url => url.startsWith('https://receipt.'))).toHaveLength(8);});+ it.each([5, 16])('caps the shared receipt slots across %i subscriptions', async count => {+ internals().dispatchLimiter = new ConcurrencyLimiter(16, 1000);+ internals().dispatchMaxQueued = 1000;+ hooksBySession['sess-m'] = Array.from({ length: count }, (_, i) =>+ createMockWebhook({+ id: `wh-many-${i}`,+ url: `https://receipt.${i}.example/hook`,+ events: ['message.receipt'],+ }),+ );+ const receipts = burst(1, 'sess-m');+ await settle(() => hangingReceipts.length >= 4);+ const other = service.dispatch('sess-o', 'message.received', {});+ await settle(() => urlsCalled().includes('https://other.example/hook'));+ try {+ expect(hangingReceipts).toHaveLength(4);+ expect(urlsCalled()).toContain('https://other.example/hook');+ } finally {+ await drain([...receipts, other]);+ }+ expect(urlsCalled().filter(url => url.startsWith('https://receipt.'))).toHaveLength(count);+ });++ it('keeps the receipt share when subscriptions change while earlier stages are active', async () => {+ internals().dispatchLimiter = new ConcurrencyLimiter(16, 1000);+ internals().dispatchMaxQueued = 1000;+ const first = burst(1);+ await settle(() => hangingReceipts.length === 1);+ hooksBySession['sess-r'].push(+ ...Array.from({ length: 4 }, (_, i) =>+ createMockWebhook({+ id: `wh-added-${i}`,+ url: `https://receipt.added-${i}.example/hook`,+ events: ['message.receipt'],+ }),+ ),+ );+ service.invalidateOptInSubscriptions('sess-r');+ const second = burst(1);+ await settle(() => hangingReceipts.length >= 4);+ try {+ expect(hangingReceipts).toHaveLength(4);+ } finally {+ await drain([...first, ...second]);+ }+ expect(urlsCalled()).toHaveLength(6);+ });++ it('drops receipt waiters at the shared session gate during shutdown', async () => {+ internals().dispatchLimiter = new ConcurrencyLimiter(16, 1000);+ internals().dispatchMaxQueued = 1000;+ (configService.get as jest.Mock).mockImplementation((key: string, def?: unknown) =>+ key === 'webhook.shutdownDrainMs' ? 0 : key === 'queue.enabled' ? false : def,+ );+ hooksBySession['sess-m'] = Array.from({ length: 16 }, (_, i) =>+ createMockWebhook({+ id: `wh-shutdown-${i}`,+ url: `https://receipt.${i}.example/hook`,+ events: ['message.receipt'],+ }),+ );+ const receipts = burst(1, 'sess-m');+ await settle(() => hangingReceipts.length === 4);+ await service.onModuleDestroy();+ await drain(receipts);++ expect(urlsCalled()).toHaveLength(4);+ expect(outboxService.open).toHaveBeenCalledTimes(4);+ expect((service as unknown as StageInternals).optInStages.size).toBe(0);+ expect((service as unknown as StageInternals).optInSessionStages.size).toBe(0);+ });+// The same with a sibling that refuses the connection: its retries, not its places, are what take time, and// the healthy webhook has everything before the first of them is sent.it("delivers a session's receipts to a healthy webhook before a refusing sibling retries", async () => {@@ -3114,6 +3186,7 @@ describe('WebhookDeliveryService', () => {await settle(() => hangingReceipts.length > 2);await drain(receipts);expect((service as unknown as StageInternals).optInStages.size).toBe(0);+ expect((service as unknown as StageInternals).optInSessionStages.size).toBe(0);}expect(urlsCalled()).toHaveLength(16);diff --git a/src/modules/webhook/webhook-delivery.service.ts b/src/modules/webhook/webhook-delivery.service.tsindex 295a56191..cb26f9266 100644--- a/src/modules/webhook/webhook-delivery.service.ts+++ b/src/modules/webhook/webhook-delivery.service.ts@@ -222,13 +222,16 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {* concurrency runs at once, shared by the session's webhooks that name the event, up to three quarters of* WEBHOOK_DISPATCH_MAX_QUEUED events wait their turn (holding only the event), and the rest are dropped* before they open an outbox row, a failure row or a hook call. A stage holds one webhook's deliveries, so- * a receiver that never answers holds only its own places, not those of its siblings. The pool never sees- * more than the running deliveries of each stage, so a flood, or a receiver that never answers, cannot- * shed or hold back what another session sends until several sessions have such receivers at once. An- * entry lives while its webhook has events in the stage; `dropped` counts those turned away since it- * was created.+ * a receiver that never answers holds only its own places, not those of its siblings. A stage keeps at+ * least one place, so the session's webhooks together could take more than a quarter: the session gate in+ * `optInSessionStages` caps what they run at once to that quarter however many webhooks name the event or+ * change while earlier stages are active. The pool therefore never sees more than a quarter from one session,+ * so a flood, or a receiver that never answers, cannot shed or hold back what another session sends until+ * several sessions have such receivers at once. An entry lives while its webhook has events in the stage;+ * `dropped` counts those turned away since it was created.*/private readonly optInStages = new Map<string, OptInStage>();+ private readonly optInSessionStages = new Map<string, { gate: ConcurrencyLimiter; stages: number }>();constructor(@InjectRepository(Webhook, 'data')@@ -295,6 +298,7 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {// Opt-in events still waiting for their stage hold nothing durable (no outbox row yet) and are best effort,// so they are let go in both modes; the ones already running finish with the rest.for (const { stage } of this.optInStages.values()) stage.close();+ for (const { gate } of this.optInSessionStages.values()) gate.close();if (!this.queueEnabled) {this.dispatchLimiter.close();for (const gate of this.degradedSessionGates.values()) gate.close();@@ -345,6 +349,11 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {named: number,): Promise<void> {const key = JSON.stringify([sessionId, webhookId]);+ let sessionStage = this.optInSessionStages.get(sessionId);+ if (!sessionStage) {+ sessionStage = { gate: new ConcurrencyLimiter(Math.floor(this.dispatchLimiter.maxCount / 4)), stages: 0 };+ this.optInSessionStages.set(sessionId, sessionStage);+ }let entry = this.optInStages.get(key);if (!entry) {// A quarter of the pool runs at once, divided among the webhooks that name the event (the limiter keeps at@@ -359,9 +368,11 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {webhookId,};this.optInStages.set(key, entry);+ sessionStage.stages++;}+ const sessionGate = sessionStage.gate;try {- await entry.stage.run(() => this.fanOut(sessionId, event, data, webhookId));+ await entry.stage.run(() => sessionGate.run(() => this.fanOut(sessionId, event, data, webhookId)));} catch (error) {// Over the stage's bound, or still waiting when shutdown closed it; anything else is not ours to hide.if (isLimiterFull(error)) this.noteOptInDrop(event, entry);@@ -391,6 +402,8 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {private releaseOptInStage(key: string, entry: OptInStage): void {if (entry.stage.activeCount > 0 || entry.stage.queuedCount > 0 || this.optInStages.get(key) !== entry) return;this.optInStages.delete(key);+ const sessionStage = this.optInSessionStages.get(entry.sessionId);+ if (sessionStage && --sessionStage.stages === 0) this.optInSessionStages.delete(entry.sessionId);if (entry.dropped === 0) return;const { sessionId, webhookId } = entry;this.logger.warn(