Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(webhook): stage opt-in events per receiving webhook
Yudhi Armyndharis47cf949eb31e
diff --git a/src/modules/webhook/webhook-delivery.service.spec.ts b/src/modules/webhook/webhook-delivery.service.spec.tsindex 2f37a4155..325f5a95d 100644--- a/src/modules/webhook/webhook-delivery.service.spec.ts+++ b/src/modules/webhook/webhook-delivery.service.spec.ts@@ -3021,6 +3021,89 @@ describe('WebhookDeliveryService', () => {expect(urlsCalled()).toHaveLength(16);});+ // The stage bounds the deliveries of one webhook, not of the event: a sibling whose receiver never answers+ // holds only its own places, so the healthy webhook of the same session keeps its pace.+ it("does not slow a session's healthy receipt webhook to the pace of a sibling that never answers", async () => {+ internals().dispatchLimiter = new ConcurrencyLimiter(16, 8); // two places per webhook+ hooksBySession['sess-m'] = [+ createMockWebhook({ id: 'wh-ok', url: 'https://ok.example/hook', events: ['message.receipt'] }),+ createMockWebhook({ id: 'wh-dead', url: 'https://receipt.dead.example/hook', events: ['message.receipt'] }),+ ];+ const receipts = burst(8, 'sess-m');+ await settle(() => urlsCalled().filter(url => url === 'https://ok.example/hook').length === 8);++ expect(urlsCalled().filter(url => url === 'https://ok.example/hook')).toHaveLength(8);+ // The dead one has its two places held and nothing more.+ expect(hangingReceipts).toHaveLength(2);++ await drain(receipts);++ expect(urlsCalled().filter(url => url.startsWith('https://receipt.'))).toHaveLength(8);+ });++ // The same with a sibling that refuses the connection: its retries, not its places, are what take time, and+ // the healthy webhook has everything before the first of them is sent.+ it("delivers a session's receipts to a healthy webhook before a refusing sibling retries", async () => {+ const original = (configService.get as jest.Mock).getMockImplementation() as (key: string) => unknown;+ (configService.get as jest.Mock).mockImplementation((key: string, def?: unknown) =>+ key === 'webhook.retryDelay' ? 400 : (original(key) ?? def),+ );+ try {+ internals().dispatchLimiter = new ConcurrencyLimiter(16, 8); // two places per webhook+ hooksBySession['sess-m'] = [+ createMockWebhook({ id: 'wh-ok', url: 'https://ok.example/hook', events: ['message.receipt'] }),+ createMockWebhook({+ id: 'wh-refusing',+ url: 'https://failing.example/hook',+ events: ['message.receipt'],+ retryCount: 2,+ }),+ ];+ const ok = () => urlsCalled().filter(url => url === 'https://ok.example/hook').length;+ const refusing = () => urlsCalled().filter(url => url === 'https://failing.example/hook').length;++ const receipts = burst(8, 'sess-m');+ await settle(() => ok() === 8);++ expect(ok()).toBe(8);+ // Two places for the refusing webhook, each used for a first attempt that was refused; no retry yet.+ expect(refusing()).toBe(2);++ await Promise.all(receipts);+ expect(refusing()).toBe(16);+ } finally {+ (configService.get as jest.Mock).mockImplementation(original);+ }+ });++ it("drops a webhook's excess receipts without touching its sibling's", async () => {+ internals().dispatchLimiter = new ConcurrencyLimiter(16, 8); // two places to run and six to wait per webhook+ hooksBySession['sess-m'] = [+ createMockWebhook({ id: 'wh-ok', url: 'https://ok.example/hook', events: ['message.receipt'] }),+ createMockWebhook({ id: 'wh-dead', url: 'https://receipt.dead.example/hook', events: ['message.receipt'] }),+ ];+ const warn = warnSpy();+ const dropsBefore = getWebhookDeliveryFailuresTotal();++ const receipts = burst(10, 'sess-m');+ await settle(() => warn.mock.calls.length > 0);+ await drain(receipts);++ // Eight of the ten are admitted to each webhook; the two over the bound are dropped for each.+ expect(urlsCalled().filter(url => url === 'https://ok.example/hook')).toHaveLength(8);+ expect(urlsCalled().filter(url => url.startsWith('https://receipt.'))).toHaveLength(8);+ expect(getWebhookDeliveryFailuresTotal() - dropsBefore).toBe(4);+ expect(warn).toHaveBeenCalledWith(+ expect.any(String),+ expect.objectContaining({+ action: 'webhook_opt_in_dropped',+ sessionId: 'sess-m',+ webhookId: 'wh-ok',+ dropped: 2,+ }),+ );+ });+it('forgets a session once its events have settled, and counts the next burst afresh', async () => {const warn = warnSpy();for (let round = 0; round < 2; round++) {@@ -3074,9 +3157,12 @@ describe('WebhookDeliveryService', () => {if (key === 'webhook.mediaInlineMaxBytes') throw new Error('config unavailable');return original(key) ?? def;});- await Promise.allSettled(burst(8));+ const thrown = await Promise.allSettled(burst(8));(configService.get as jest.Mock).mockImplementation(original);+ // It is not the stage's to hide: every dispatch reports it.+ expect(thrown.map(result => result.status)).toEqual(Array(8).fill('rejected'));+const receipts = burst(2);await settle(() => hangingReceipts.length > 1);diff --git a/src/modules/webhook/webhook-delivery.service.ts b/src/modules/webhook/webhook-delivery.service.tsindex 80ea4911f..9f6cc5223 100644--- a/src/modules/webhook/webhook-delivery.service.ts+++ b/src/modules/webhook/webhook-delivery.service.ts@@ -139,6 +139,20 @@ const isLimiterClosed = (error: unknown): boolean =>const isLimiterFull = (error: unknown): boolean =>error instanceof Error && error.message === 'ConcurrencyLimiter queue full';+/**+ * The stage one webhook's opt-in events pass (`webhookId` is null when the webhooks that name the event could+ * not be listed, and one stage then serves the whole session); `dropped` counts the events it turned away.+ */+interface OptInStage {+ stage: ConcurrencyLimiter;+ dropped: number;+ sessionId: string;+ webhookId: string | null;+}++const describeOptInStage = ({ sessionId, webhookId }: OptInStage): string =>+ webhookId === null ? `session ${sessionId}` : `webhook ${webhookId} of session ${sessionId}`;+/*** The replay's retries were stopped by its aborted signal, in a backoff or just before a retry POST.* A POST that fails after the abort is not this: on the last attempt it is a terminal failure and is@@ -194,24 +208,27 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {*/private readonly locallyPending = new Map<string, number>();/**- * Per session, the opt-in events (WILDCARD_EXCLUDED_EVENTS) that active webhooks name, with how many- * name each. The promise is cached, not its result, so a burst of receipts on a cold entry shares one lookup.+ * Per session, the opt-in events (WILDCARD_EXCLUDED_EVENTS) that active webhooks name, with the ids of the+ * webhooks that name each. The promise is cached, not its result, so a burst of receipts on a cold entry+ * shares one lookup.*/private readonly optInSubscriptions = new Map<string,- { named: Promise<ReadonlyMap<string, number>>; expiresAt: number }+ { named: Promise<ReadonlyMap<string, readonly string[]>>; expiresAt: number }>();/**- * Per session, the stage its opt-in events (WILDCARD_EXCLUDED_EVENTS) pass before the shared dispatch+ * Per webhook, the stage its opt-in events (WILDCARD_EXCLUDED_EVENTS) pass before the shared dispatch* pool. One flush of buffered receipts is thousands of dispatches in a tick: a quarter of the dispatch- * concurrency runs at once, shared by the webhooks that name the event since one event is a delivery to- * each, up to three quarters of WEBHOOK_DISPATCH_MAX_QUEUED events wait their turn (holding only the- * event), and the rest are dropped before they open an outbox row, a failure row or a hook call. The pool- * never sees more than a session's running deliveries, so a flood, or a receiver that never answers,- * cannot shed or hold back what another session sends. An entry lives while its session has events in- * the stage; `dropped` counts those turned away since it was created.+ * concurrency runs at once, shared by the session's webhooks that name the event, up to three quarters of+ * WEBHOOK_DISPATCH_MAX_QUEUED events wait their turn (holding only the event), and the rest are dropped+ * before they open an outbox row, a failure row or a hook call. A stage holds one webhook's deliveries, so+ * a receiver that never answers holds only its own places, not those of its siblings. The pool never sees+ * more than the running deliveries of each stage, so a flood, or a receiver that never answers, cannot+ * shed or hold back what another session sends until several sessions have such receivers at once. An+ * entry lives while its webhook has events in the stage; `dropped` counts those turned away since it+ * was created.*/- private readonly optInStages = new Map<string, { stage: ConcurrencyLimiter; dropped: number }>();+ private readonly optInStages = new Map<string, OptInStage>();constructor(@InjectRepository(Webhook, 'data')@@ -306,12 +323,29 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {}async dispatch(sessionId: string, event: string, data: Record<string, unknown>): Promise<void> {- if (!WILDCARD_EXCLUDED_EVENTS.includes(event)) return this.fanOut(sessionId, event, data);+ if (!WILDCARD_EXCLUDED_EVENTS.includes(event)) return this.fanOut(sessionId, event, data, null);// An opt-in event fires per recipient (about 2,000 per message to a 1,000-member group): skip it// before the per-dispatch webhook read when no active webhook of the session names it.- const named = await this.webhooksNamingOptInEvent(sessionId, event);- if (named === 0) return;- let entry = this.optInStages.get(sessionId);+ const webhookIds = await this.webhooksNamingOptInEvent(sessionId, event);+ // One event is a delivery to each of them; each delivery waits in the stage of its own webhook. A single+ // webhook is the common case and passes straight through: what waits in a stage is held for a flush.+ if (webhookIds.length === 1) return this.dispatchThroughOptInStage(sessionId, event, data, webhookIds[0], 1);+ const settled = await Promise.allSettled(+ webhookIds.map(webhookId => this.dispatchThroughOptInStage(sessionId, event, data, webhookId, webhookIds.length)),+ );+ const failed = settled.find((result): result is PromiseRejectedResult => result.status === 'rejected');+ if (failed) throw failed.reason;+ }++ private async dispatchThroughOptInStage(+ sessionId: string,+ event: string,+ data: Record<string, unknown>,+ webhookId: string | null,+ named: number,+ ): Promise<void> {+ const key = JSON.stringify([sessionId, webhookId]);+ let entry = this.optInStages.get(key);if (!entry) {// A quarter of the pool runs at once, divided among the webhooks that name the event (the limiter keeps at// least one event running), and three quarters of the park bound wait, at least one place.@@ -321,49 +355,71 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {Math.max(1, Math.floor((this.dispatchMaxQueued * 3) / 4)),),dropped: 0,+ sessionId,+ webhookId,};- this.optInStages.set(sessionId, entry);+ this.optInStages.set(key, entry);}try {- await entry.stage.run(() => this.fanOut(sessionId, event, data));+ await entry.stage.run(() => this.fanOut(sessionId, event, data, webhookId));} catch (error) {// Over the stage's bound, or still waiting when shutdown closed it; anything else is not ours to hide.- if (isLimiterFull(error)) this.noteOptInDrop(sessionId, event, entry);+ if (isLimiterFull(error)) this.noteOptInDrop(event, entry);else if (!isLimiterClosed(error)) throw error;} finally {- this.releaseOptInStage(sessionId, entry);+ this.releaseOptInStage(key, entry);}}/**- * An opt-in event past its session's bound: counted with the other deliveries that were never sent, and- * announced once per burst, when the session starts dropping. It leaves no row to replay, by design.+ * An opt-in event past its webhook's bound: counted with the other deliveries that were never sent, and+ * announced once per burst, when the webhook starts dropping. It leaves no row to replay, by design.*/- private noteOptInDrop(sessionId: string, event: string, entry: { dropped: number }): void {+ private noteOptInDrop(event: string, entry: OptInStage): void {incrementWebhookDeliveryFailures();if (entry.dropped++ > 0) return;- this.logger.warn(`Dropping ${event} events of session ${sessionId}: its queue of opt-in events is full`, {+ const { sessionId, webhookId } = entry;+ this.logger.warn(`Dropping ${event} events of ${describeOptInStage(entry)}: its queue of opt-in events is full`, {sessionId,+ webhookId: webhookId ?? undefined,event,action: 'webhook_opt_in_dropping',});}- /** Forget a session's stage once it is idle, reporting how many events it turned away. */- private releaseOptInStage(sessionId: string, entry: { stage: ConcurrencyLimiter; dropped: number }): void {- if (entry.stage.activeCount > 0 || entry.stage.queuedCount > 0 || this.optInStages.get(sessionId) !== entry) return;- this.optInStages.delete(sessionId);+ /** Forget a webhook's stage once it is idle, reporting how many events it turned away. */+ private releaseOptInStage(key: string, entry: OptInStage): void {+ if (entry.stage.activeCount > 0 || entry.stage.queuedCount > 0 || this.optInStages.get(key) !== entry) return;+ this.optInStages.delete(key);if (entry.dropped === 0) return;- this.logger.warn(`Dropped ${entry.dropped} opt-in events of session ${sessionId} while its queue was full`, {- sessionId,- dropped: entry.dropped,- action: 'webhook_opt_in_dropped',- });+ const { sessionId, webhookId } = entry;+ this.logger.warn(+ `Dropped ${entry.dropped} opt-in events of ${describeOptInStage(entry)} while its queue was full`,+ {+ sessionId,+ webhookId: webhookId ?? undefined,+ dropped: entry.dropped,+ action: 'webhook_opt_in_dropped',+ },+ );}- private async fanOut(sessionId: string, event: string, data: Record<string, unknown>): Promise<void> {+ /**+ * Deliver an event to the session's matching webhooks, or only to the webhook `only` names. `only` has no+ * default value: a parameter list with one keeps the caller's `data` reachable past the reassignment below.+ */+ private async fanOut(+ sessionId: string,+ event: string,+ data: Record<string, unknown>,+ only: string | null,+ ): Promise<void> {const webhooks = await this.loadActiveWebhooks(sessionId, event);- const matchingWebhooks = this.filterMatchingWebhooks(webhooks, event, data);+ const matchingWebhooks = this.filterMatchingWebhooks(+ only === null ? webhooks : webhooks.filter(webhook => webhook.id === only),+ event,+ data,+ );// Base idempotency key for this event occurrence. occurredAt is captured once here and reused for// every retry of this dispatch, so recurring lifecycle events get a distinct-per-occurrence key@@ -401,11 +457,11 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {}/**- * How many active webhooks of the session name `event`, read at most once per- * OPT_IN_SUBSCRIPTIONS_TTL_MS. A failed lookup is not cached and counts as one, which lets the event- * through, so dispatch's own lookup reports the failure as before.+ * The ids of the session's active webhooks that name `event`, read at most once per+ * OPT_IN_SUBSCRIPTIONS_TTL_MS. A failed lookup is not cached and answers `[null]`, one delivery to every+ * webhook, which lets the event through so dispatch's own lookup reports the failure as before.*/- private async webhooksNamingOptInEvent(sessionId: string, event: string): Promise<number> {+ private async webhooksNamingOptInEvent(sessionId: string, event: string): Promise<Array<string | null>> {const now = Date.now();let entry = this.optInSubscriptions.get(sessionId);if (!entry || entry.expiresAt <= now) {@@ -413,11 +469,14 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {named: this.webhookRepository.find({ select: { id: true, events: true }, where: { sessionId, active: true } }).then(rows => {- const named = new Map<string, number>();+ const named = new Map<string, string[]>();for (const row of rows) {if (!Array.isArray(row.events)) continue;for (const e of new Set(row.events)) {- if (WILDCARD_EXCLUDED_EVENTS.includes(e)) named.set(e, (named.get(e) ?? 0) + 1);+ if (!WILDCARD_EXCLUDED_EVENTS.includes(e)) continue;+ const ids = named.get(e);+ if (ids) ids.push(row.id);+ else named.set(e, [row.id]);}}return named;@@ -432,9 +491,9 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {entry = created;}try {- return (await entry.named).get(event) ?? 0;+ return [...((await entry.named).get(event) ?? [])];} catch {- return 1;+ return [null];}}