Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(webhook): cap a session's receipt deliveries across webhooks

Yudhi Armyndharisbdc0c05e86b1
diff --git a/src/modules/webhook/webhook-delivery.service.spec.ts b/src/modules/webhook/webhook-delivery.service.spec.ts
index 8e3193a5c..df09afdba 100644
--- a/src/modules/webhook/webhook-delivery.service.spec.ts
+++ b/src/modules/webhook/webhook-delivery.service.spec.ts
@@ -2864,7 +2864,7 @@ describe('WebhookDeliveryService', () => {
// turn, and the rest are dropped. The pool therefore never holds more than a session's running ones, which keeps
// a flood or a dead receiver of one session from shedding or holding back what another session sends.
describe('a burst of opt-in events', () => {
- type StageInternals = { optInStages: Map<string, unknown> };
+ type StageInternals = { optInStages: Map<string, unknown>; optInSessionStages: Map<string, unknown> };
const receipt = (i: number) => ({ id: 'OUT1', participant: `${i}@c.us`, status: 'read' });
const hangingReceipts: Array<() => void> = [];
const warnSpy = () =>
@@ -3044,6 +3044,78 @@ describe('WebhookDeliveryService', () => {
expect(urlsCalled().filter(url => url.startsWith('https://receipt.'))).toHaveLength(8);
});
+ it.each([5, 16])('caps the shared receipt slots across %i subscriptions', async count => {
+ internals().dispatchLimiter = new ConcurrencyLimiter(16, 1000);
+ internals().dispatchMaxQueued = 1000;
+ hooksBySession['sess-m'] = Array.from({ length: count }, (_, i) =>
+ createMockWebhook({
+ id: `wh-many-${i}`,
+ url: `https://receipt.${i}.example/hook`,
+ events: ['message.receipt'],
+ }),
+ );
+ const receipts = burst(1, 'sess-m');
+ await settle(() => hangingReceipts.length >= 4);
+ const other = service.dispatch('sess-o', 'message.received', {});
+ await settle(() => urlsCalled().includes('https://other.example/hook'));
+ try {
+ expect(hangingReceipts).toHaveLength(4);
+ expect(urlsCalled()).toContain('https://other.example/hook');
+ } finally {
+ await drain([...receipts, other]);
+ }
+ expect(urlsCalled().filter(url => url.startsWith('https://receipt.'))).toHaveLength(count);
+ });
+
+ it('keeps the receipt share when subscriptions change while earlier stages are active', async () => {
+ internals().dispatchLimiter = new ConcurrencyLimiter(16, 1000);
+ internals().dispatchMaxQueued = 1000;
+ const first = burst(1);
+ await settle(() => hangingReceipts.length === 1);
+ hooksBySession['sess-r'].push(
+ ...Array.from({ length: 4 }, (_, i) =>
+ createMockWebhook({
+ id: `wh-added-${i}`,
+ url: `https://receipt.added-${i}.example/hook`,
+ events: ['message.receipt'],
+ }),
+ ),
+ );
+ service.invalidateOptInSubscriptions('sess-r');
+ const second = burst(1);
+ await settle(() => hangingReceipts.length >= 4);
+ try {
+ expect(hangingReceipts).toHaveLength(4);
+ } finally {
+ await drain([...first, ...second]);
+ }
+ expect(urlsCalled()).toHaveLength(6);
+ });
+
+ it('drops receipt waiters at the shared session gate during shutdown', async () => {
+ internals().dispatchLimiter = new ConcurrencyLimiter(16, 1000);
+ internals().dispatchMaxQueued = 1000;
+ (configService.get as jest.Mock).mockImplementation((key: string, def?: unknown) =>
+ key === 'webhook.shutdownDrainMs' ? 0 : key === 'queue.enabled' ? false : def,
+ );
+ hooksBySession['sess-m'] = Array.from({ length: 16 }, (_, i) =>
+ createMockWebhook({
+ id: `wh-shutdown-${i}`,
+ url: `https://receipt.${i}.example/hook`,
+ events: ['message.receipt'],
+ }),
+ );
+ const receipts = burst(1, 'sess-m');
+ await settle(() => hangingReceipts.length === 4);
+ await service.onModuleDestroy();
+ await drain(receipts);
+
+ expect(urlsCalled()).toHaveLength(4);
+ expect(outboxService.open).toHaveBeenCalledTimes(4);
+ expect((service as unknown as StageInternals).optInStages.size).toBe(0);
+ expect((service as unknown as StageInternals).optInSessionStages.size).toBe(0);
+ });
+
// The same with a sibling that refuses the connection: its retries, not its places, are what take time, and
// the healthy webhook has everything before the first of them is sent.
it("delivers a session's receipts to a healthy webhook before a refusing sibling retries", async () => {
@@ -3114,6 +3186,7 @@ describe('WebhookDeliveryService', () => {
await settle(() => hangingReceipts.length > 2);
await drain(receipts);
expect((service as unknown as StageInternals).optInStages.size).toBe(0);
+ expect((service as unknown as StageInternals).optInSessionStages.size).toBe(0);
}
expect(urlsCalled()).toHaveLength(16);
diff --git a/src/modules/webhook/webhook-delivery.service.ts b/src/modules/webhook/webhook-delivery.service.ts
index 295a56191..cb26f9266 100644
--- a/src/modules/webhook/webhook-delivery.service.ts
+++ b/src/modules/webhook/webhook-delivery.service.ts
@@ -222,13 +222,16 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
* concurrency runs at once, shared by the session's webhooks that name the event, up to three quarters of
* WEBHOOK_DISPATCH_MAX_QUEUED events wait their turn (holding only the event), and the rest are dropped
* before they open an outbox row, a failure row or a hook call. A stage holds one webhook's deliveries, so
- * a receiver that never answers holds only its own places, not those of its siblings. The pool never sees
- * more than the running deliveries of each stage, so a flood, or a receiver that never answers, cannot
- * shed or hold back what another session sends until several sessions have such receivers at once. An
- * entry lives while its webhook has events in the stage; `dropped` counts those turned away since it
- * was created.
+ * a receiver that never answers holds only its own places, not those of its siblings. A stage keeps at
+ * least one place, so the session's webhooks together could take more than a quarter: the session gate in
+ * `optInSessionStages` caps what they run at once to that quarter however many webhooks name the event or
+ * change while earlier stages are active. The pool therefore never sees more than a quarter from one session,
+ * so a flood, or a receiver that never answers, cannot shed or hold back what another session sends until
+ * several sessions have such receivers at once. An entry lives while its webhook has events in the stage;
+ * `dropped` counts those turned away since it was created.
*/
private readonly optInStages = new Map<string, OptInStage>();
+ private readonly optInSessionStages = new Map<string, { gate: ConcurrencyLimiter; stages: number }>();
constructor(
@InjectRepository(Webhook, 'data')
@@ -295,6 +298,7 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
// Opt-in events still waiting for their stage hold nothing durable (no outbox row yet) and are best effort,
// so they are let go in both modes; the ones already running finish with the rest.
for (const { stage } of this.optInStages.values()) stage.close();
+ for (const { gate } of this.optInSessionStages.values()) gate.close();
if (!this.queueEnabled) {
this.dispatchLimiter.close();
for (const gate of this.degradedSessionGates.values()) gate.close();
@@ -345,6 +349,11 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
named: number,
): Promise<void> {
const key = JSON.stringify([sessionId, webhookId]);
+ let sessionStage = this.optInSessionStages.get(sessionId);
+ if (!sessionStage) {
+ sessionStage = { gate: new ConcurrencyLimiter(Math.floor(this.dispatchLimiter.maxCount / 4)), stages: 0 };
+ this.optInSessionStages.set(sessionId, sessionStage);
+ }
let entry = this.optInStages.get(key);
if (!entry) {
// A quarter of the pool runs at once, divided among the webhooks that name the event (the limiter keeps at
@@ -359,9 +368,11 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
webhookId,
};
this.optInStages.set(key, entry);
+ sessionStage.stages++;
}
+ const sessionGate = sessionStage.gate;
try {
- await entry.stage.run(() => this.fanOut(sessionId, event, data, webhookId));
+ await entry.stage.run(() => sessionGate.run(() => this.fanOut(sessionId, event, data, webhookId)));
} catch (error) {
// Over the stage's bound, or still waiting when shutdown closed it; anything else is not ours to hide.
if (isLimiterFull(error)) this.noteOptInDrop(event, entry);
@@ -391,6 +402,8 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {
private releaseOptInStage(key: string, entry: OptInStage): void {
if (entry.stage.activeCount > 0 || entry.stage.queuedCount > 0 || this.optInStages.get(key) !== entry) return;
this.optInStages.delete(key);
+ const sessionStage = this.optInSessionStages.get(entry.sessionId);
+ if (sessionStage && --sessionStage.stages === 0) this.optInSessionStages.delete(entry.sessionId);
if (entry.dropped === 0) return;
const { sessionId, webhookId } = entry;
this.logger.warn(