Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(webhook): bound opt-in events per session
Yudhi Armyndharisc58ecfdd5d31
diff --git a/src/common/metrics/webhook-delivery-metrics.ts b/src/common/metrics/webhook-delivery-metrics.tsindex e547d2ca8..8f6afc3e1 100644--- a/src/common/metrics/webhook-delivery-metrics.ts+++ b/src/common/metrics/webhook-delivery-metrics.ts@@ -3,7 +3,9 @@* exhausted, deliveries never sent (shed, refused at shutdown, rejected before sending), and direct* deliveries stopped by shutdown between retries after earlier attempts were sent. It is* incremented once per row inserted into the durable `webhook_delivery_failures` dead-letter table;- * a later replay may replace or remove that row, and the counter keeps the failure. Kept as a plain+ * a later replay may replace or remove that row, and the counter keeps the failure. It is also+ * incremented once for each opt-in event dropped past the bound of its session's queue, which files+ * no row because there is nothing to replay. Kept as a plain* in-process counter rather than a `COUNT(*)` over that table because the table is pruned on a* retention schedule, which would make its count non-monotonic and therefore invalid as a* Prometheus `counter` (a prune would look like a counter reset to `rate()`/`increase()`). Andiff --git a/src/modules/metrics/metrics.service.ts b/src/modules/metrics/metrics.service.tsindex 292be7820..8ed39f400 100644--- a/src/modules/metrics/metrics.service.ts+++ b/src/modules/metrics/metrics.service.ts@@ -221,7 +221,7 @@ export class MetricsService implements OnModuleDestroy {}lines.push(- '# HELP openwa_webhook_delivery_failures_total Webhook delivery failures recorded since process start: retries exhausted, never sent (shed, refused at shutdown, rejected before sending), or stopped by shutdown between direct retries.',+ '# HELP openwa_webhook_delivery_failures_total Webhook delivery failures recorded since process start: retries exhausted, never sent (shed, refused at shutdown, rejected before sending, or an opt-in event dropped past its session bound), or stopped by shutdown between direct retries.',);lines.push('# TYPE openwa_webhook_delivery_failures_total counter');lines.push(`openwa_webhook_delivery_failures_total ${getWebhookDeliveryFailuresTotal()}`);diff --git a/src/modules/webhook/webhook-delivery.service.spec.ts b/src/modules/webhook/webhook-delivery.service.spec.tsindex 0818029a6..ddf085be8 100644--- a/src/modules/webhook/webhook-delivery.service.spec.ts+++ b/src/modules/webhook/webhook-delivery.service.spec.ts@@ -1780,6 +1780,38 @@ describe('WebhookDeliveryService', () => {expect(webhookQueue.add).toHaveBeenCalledWith('webhook-wh-named', expect.anything(), expect.anything());});+ // One flush of receipts from a session, with the default dispatch settings: 500 are all enqueued, and of 2000+ // the 4 that run and the 750 that wait are, the others being dropped and counted.+ it.each([+ [500, 500],+ [2000, 754],+ ])(+ 'enqueues %i receipts of one session in a tick as %i jobs, and counts the rest as dropped',+ async (size, queued) => {+ const queueService = await buildQueueService((key: string, def?: unknown) =>+ key === 'queue.enabled' ? true : def,+ );+ (repository.find as jest.Mock).mockResolvedValue([createMockWebhook({ events: ['message.receipt'] })]);+ (hookManager.execute as jest.Mock).mockResolvedValue({ continue: true, data: {} });+ const dropsBefore = getWebhookDeliveryFailuresTotal();++ await Promise.all(+ Array.from({ length: size }, (_, i) =>+ queueService.dispatch('sess-1', 'message.receipt', {+ id: 'OUT1',+ participant: `${i}@c.us`,+ status: 'read',+ }),+ ),+ );++ expect(webhookQueue.add).toHaveBeenCalledTimes(queued);+ expect(outboxService.open).toHaveBeenCalledTimes(queued);+ expect(insertedFailures).toEqual([]);+ expect(getWebhookDeliveryFailuresTotal() - dropsBefore).toBe(size - queued);+ },+ );+it('uses one direct POST for operator redrive even with the queue enabled', async () => {const queueService = await buildQueueService((key: string, def?: unknown) =>key === 'queue.enabled' ? true : def,@@ -2823,15 +2855,19 @@ describe('WebhookDeliveryService', () => {await a;});- // An opt-in event fires once per recipient and status, so one flush of buffered receipts can be- // thousands of dispatches in a single tick. They must not spend the budget every session shares.+ // An opt-in event fires once per recipient and status, so one flush of buffered receipts can be thousands of+ // dispatches in a tick. Each session passes a stage of its own before the shared pool: a quarter of the+ // dispatch concurrency of its events run at once, three quarters of WEBHOOK_DISPATCH_MAX_QUEUED more wait their+ // turn, and the rest are dropped. The pool therefore never holds more than a session's running ones, which keeps+ // a flood or a dead receiver of one session from shedding or holding back what another session sends.describe('a burst of opt-in events', () => {+ type StageInternals = { optInStages: Map<string, unknown> };const receipt = (i: number) => ({ id: 'OUT1', participant: `${i}@c.us`, status: 'read' });const hangingReceipts: Array<() => void> = [];const warnSpy = () =>jest.spyOn((service as unknown as { logger: { warn: (...args: unknown[]) => void } }).logger, 'warn');- const burst = (size: number) =>- Array.from({ length: size }, (_, i) => service.dispatch('sess-r', 'message.receipt', receipt(i)));+ const burst = (size: number, sessionId = 'sess-r') =>+ Array.from({ length: size }, (_, i) => service.dispatch(sessionId, 'message.receipt', receipt(i)));// Answers the receipts' receiver until every dispatch has settled, parked ones included.const drain = async (dispatches: Array<Promise<void>>): Promise<void> => {let settled = false;@@ -2844,48 +2880,65 @@ describe('WebhookDeliveryService', () => {beforeEach(() => {hangingReceipts.length = 0;- // 4 slots and a park bound of 8, so the opt-in allowance, a quarter of 8, is 2 events at a time.- internals().dispatchLimiter = new ConcurrencyLimiter(4, 8);+ // 8 slots and a park bound of 8: a session's receipts run 2 at a time (a quarter of 8) and 6 more wait+ // (three quarters of 8), so 8 of a burst are admitted.+ internals().dispatchLimiter = new ConcurrencyLimiter(8, 8);internals().dispatchMaxQueued = 8;hooksBySession['sess-r'] = [createMockWebhook({ id: 'wh-r', url: 'https://receipt.example/hook', events: ['message.receipt'] }),];+ hooksBySession['sess-q'] = [+ createMockWebhook({ id: 'wh-q', url: 'https://quiet.example/hook', events: ['message.receipt'] }),+ ];+ hooksBySession['sess-f'] = [+ createMockWebhook({+ id: 'wh-f',+ url: 'https://failing.example/hook',+ events: ['message.receipt'],+ retryCount: 2,+ }),+ ];hooksBySession['sess-o'] = [createMockWebhook({ id: 'wh-o', url: 'https://other.example/hook', events: ['message.received'] }),];- mockFetch.mockImplementation((url: string) =>- url.startsWith('https://receipt.')- ? new Promise(resolve => hangingReceipts.push(() => resolve({ ok: true, status: 200 })))- : Promise.resolve({ ok: true, status: 200 }),- );+ mockFetch.mockImplementation((url: string) => {+ if (url.startsWith('https://receipt.')) {+ return new Promise(resolve => hangingReceipts.push(() => resolve({ ok: true, status: 200 })));+ }+ if (url.startsWith('https://failing.')) return Promise.reject(new Error('ECONNREFUSED'));+ return Promise.resolve({ ok: true, status: 200 });+ });});- it('leaves the shared dispatch budget to the other sessions and records no failure for the excess', async () => {+ it('runs a quarter of the dispatch concurrency at once, queues three quarters of the park bound behind it, and drops the rest', async () => {+ const dropsBefore = getWebhookDeliveryFailuresTotal();const receipts = burst(14);- await settle(() => hangingReceipts.length > 1);+ await settle(() => hangingReceipts.length > 2);- await service.dispatch('sess-o', 'message.received', {});+ // Only the running ones have read the webhooks and opened an outbox row; the queued ones cost nothing yet.+ expect(hangingReceipts).toHaveLength(2);+ expect(outboxService.open).toHaveBeenCalledTimes(2);- // The other session's event went out while the receipts were still waiting on their receiver.- expect(urlsCalled().filter(url => url.startsWith('https://other.'))).toHaveLength(1);+ await drain(receipts);++ expect(urlsCalled()).toHaveLength(8);+ expect(outboxService.open).toHaveBeenCalledTimes(8);+ expect(outboxService.close).toHaveBeenCalledTimes(8);+ // A dropped event leaves no failure row, outbox row or hook call: it is only counted, and logged.expect(insertedFailures).toEqual([]);expect(hookManager.execute).not.toHaveBeenCalledWith('webhook:error', expect.anything(), expect.anything());- // Only the admitted receipts and the other session's event ever opened an outbox row.- expect(outboxService.open).toHaveBeenCalledTimes(3);-- await drain(receipts);- expect(outboxService.close).toHaveBeenCalledTimes(3);+ expect(getWebhookDeliveryFailuresTotal() - dropsBefore).toBe(6);});- it('warns when it starts dropping, and again with the count once the admitted events are done', async () => {+ it('warns when a session starts dropping, and again with the count once its events are done', async () => {const warn = warnSpy();const receipts = burst(14);- await settle(() => hangingReceipts.length > 1);+ await settle(() => hangingReceipts.length > 2);expect(warn).toHaveBeenCalledTimes(1);expect(warn).toHaveBeenCalledWith(expect.any(String),- expect.objectContaining({ action: 'webhook_opt_in_dropping' }),+ expect.objectContaining({ action: 'webhook_opt_in_dropping', sessionId: 'sess-r' }),);await drain(receipts);@@ -2893,20 +2946,96 @@ describe('WebhookDeliveryService', () => {expect(warn).toHaveBeenCalledTimes(2);expect(warn).toHaveBeenLastCalledWith(expect.any(String),- expect.objectContaining({ action: 'webhook_opt_in_dropped', dropped: 12 }),+ expect.objectContaining({ action: 'webhook_opt_in_dropped', sessionId: 'sess-r', dropped: 6 }),);});- it('admits opt-in events again once the admitted ones are done, and warns again on the next burst', async () => {+ it('drops and logs nothing for a burst its stage holds', async () => {+ const warn = warnSpy();+ const dropsBefore = getWebhookDeliveryFailuresTotal();++ await drain(burst(8));++ expect(urlsCalled()).toHaveLength(8);+ expect(warn).not.toHaveBeenCalled();+ expect(getWebhookDeliveryFailuresTotal()).toBe(dropsBefore);+ });++ it("leaves another session's ordinary events and receipts alone while one session holds its stage", async () => {+ const receipts = burst(14);+ await settle(() => hangingReceipts.length > 2);++ // The receiver of sess-r never answers, and its stage is full.+ await service.dispatch('sess-o', 'message.received', {});+ await service.dispatch('sess-q', 'message.receipt', receipt(1));++ expect(urlsCalled()).toEqual(+ expect.arrayContaining(['https://other.example/hook', 'https://quiet.example/hook']),+ );+ expect(insertedFailures).toEqual([]);+ expect(hookManager.execute).not.toHaveBeenCalledWith('webhook:error', expect.anything(), expect.anything());+ await drain(receipts);+ });++ it("delivers another session's receipt while one session holds its stage with a receiver that keeps failing", async () => {+ const failing = burst(14, 'sess-f');+ let failingSettled = false;+ void Promise.all(failing).then(() => (failingSettled = true));+ await settle(() => urlsCalled().length > 0);++ await service.dispatch('sess-q', 'message.receipt', receipt(1));++ // It went out while the failing session was still working through its retries.+ expect(urlsCalled()).toContain('https://quiet.example/hook');+ expect(failingSettled).toBe(false);+ await Promise.all(failing);+ });++ it('forgets a session once its events have settled, and counts the next burst afresh', async () => {const warn = warnSpy();for (let round = 0; round < 2; round++) {- const receipts = burst(5);- await settle(() => hangingReceipts.length > 1);+ const receipts = burst(10);+ await settle(() => hangingReceipts.length > 2);await drain(receipts);+ expect((service as unknown as StageInternals).optInStages.size).toBe(0);}- expect(urlsCalled()).toHaveLength(4);- expect(warn.mock.calls.filter(([, meta]) => (meta as { dropped?: number }).dropped === 3)).toHaveLength(2);+ expect(urlsCalled()).toHaveLength(16);+ expect(warn.mock.calls.filter(([, meta]) => (meta as { dropped?: number }).dropped === 2)).toHaveLength(2);+ });++ it('keeps at least one place to run and one to wait however small the pool', async () => {+ internals().dispatchLimiter = new ConcurrencyLimiter(1, 0);+ internals().dispatchMaxQueued = 0;+ const dropsBefore = getWebhookDeliveryFailuresTotal();++ await drain(burst(3));++ expect(urlsCalled()).toHaveLength(2);+ expect(getWebhookDeliveryFailuresTotal() - dropsBefore).toBe(1);+ });++ it('drops the events still waiting at shutdown without counting or recording them', async () => {+ (configService.get as jest.Mock).mockImplementation(<T>(key: string, def?: T): T | boolean | number =>+ key === 'webhook.shutdownDrainMs' ? 100 : key === 'queue.enabled' ? false : (def as T),+ );+ const warn = warnSpy();+ const dropsBefore = getWebhookDeliveryFailuresTotal();+ const receipts = burst(14);+ await settle(() => hangingReceipts.length > 2);++ await service.onModuleDestroy();+ await drain(receipts);++ // The two that were running finish; the six that waited are gone, and only the six over the bound were+ // counted and reported.+ expect(urlsCalled()).toHaveLength(2);+ expect(getWebhookDeliveryFailuresTotal() - dropsBefore).toBe(6);+ expect(insertedFailures).toEqual([]);+ expect(warn).toHaveBeenLastCalledWith(+ expect.any(String),+ expect.objectContaining({ action: 'webhook_opt_in_dropped', dropped: 6 }),+ );});it('gives its place back when a dispatch throws', async () => {@@ -2915,7 +3044,7 @@ describe('WebhookDeliveryService', () => {if (key === 'webhook.mediaInlineMaxBytes') throw new Error('config unavailable');return original(key) ?? def;});- await Promise.allSettled(burst(2));+ await Promise.allSettled(burst(8));(configService.get as jest.Mock).mockImplementation(original);const receipts = burst(2);@@ -2931,6 +3060,37 @@ describe('WebhookDeliveryService', () => {expect(urlsCalled()).toHaveLength(6);expect(insertedFailures).toEqual([]);});++ // The sizes an operator gets by default: a flush of 500 receipts is delivered in full, and a flush of 2000+ // delivers the 4 that run and the 750 that wait.+ describe('with the default dispatch settings', () => {+ beforeEach(() => {+ internals().dispatchLimiter = new ConcurrencyLimiter(16, 1000);+ internals().dispatchMaxQueued = 1000;+ mockFetch.mockResolvedValue({ ok: true, status: 200 });+ });++ it('delivers a flush of 500 receipts from one session in full', async () => {+ const dropsBefore = getWebhookDeliveryFailuresTotal();++ await Promise.all(burst(500));++ expect(urlsCalled()).toHaveLength(500);+ expect(insertedFailures).toEqual([]);+ expect(getWebhookDeliveryFailuresTotal()).toBe(dropsBefore);+ });++ it('delivers at least 750 of a flush of 2000 and counts every other one as dropped', async () => {+ const dropsBefore = getWebhookDeliveryFailuresTotal();++ await Promise.all(burst(2000));++ expect(urlsCalled()).toHaveLength(754);+ expect(outboxService.open).toHaveBeenCalledTimes(754);+ expect(insertedFailures).toEqual([]);+ expect(getWebhookDeliveryFailuresTotal() - dropsBefore).toBe(2000 - 754);+ });+ });});});diff --git a/src/modules/webhook/webhook-delivery.service.ts b/src/modules/webhook/webhook-delivery.service.tsindex aa3009e9c..5b5530730 100644--- a/src/modules/webhook/webhook-delivery.service.ts+++ b/src/modules/webhook/webhook-delivery.service.ts@@ -135,6 +135,10 @@ class UnrecordedWebhookFailure extends Error {}const isLimiterClosed = (error: unknown): boolean =>error instanceof Error && error.message === 'ConcurrencyLimiter closed';+/** The limiter refused the task because it already runs and queues as many as it admits. */+const isLimiterFull = (error: unknown): boolean =>+ error instanceof Error && error.message === 'ConcurrencyLimiter queue full';+/*** The replay's retries were stopped by its aborted signal, in a backoff or just before a retry POST.* A POST that fails after the abort is not this: on the last attempt it is a terminal failure and is@@ -195,13 +199,15 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {*/private readonly optInSubscriptions = new Map<string, { events: Promise<ReadonlySet<string>>; expiresAt: number }>();/**- * Opt-in events admitted by dispatch() and not yet settled. One flush of buffered receipts is- * thousands of dispatches in a tick, so they hold at most a quarter of the park budget, as a session- * gate does; the excess is dropped before it opens an outbox row, a failure row or a hook call.+ * Per session, the stage its opt-in events (WILDCARD_EXCLUDED_EVENTS) pass before the shared dispatch+ * pool. One flush of buffered receipts is thousands of dispatches in a tick: a quarter of the dispatch+ * concurrency of them run at once, up to three quarters of WEBHOOK_DISPATCH_MAX_QUEUED wait their turn+ * (holding only the event), and the rest are dropped before they open an outbox row, a failure row or a+ * hook call. The pool never sees more than the running ones, so a flood, or a receiver that never+ * answers, cannot shed or hold back what another session sends. An entry lives while its session has+ * events in the stage; `dropped` counts those turned away since it was created.*/- private optInPending = 0;- /** Opt-in events dropped since the allowance last emptied, reported when it does. */- private optInDropped = 0;+ private readonly optInStages = new Map<string, { stage: ConcurrencyLimiter; dropped: number }>();constructor(@InjectRepository(Webhook, 'data')@@ -265,6 +271,9 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {* so the bound limits this hook only; the whole teardown is the sum in shutdown-budget.ts.*/async onModuleDestroy(): Promise<void> {+ // Opt-in events still waiting for their stage hold nothing durable (no outbox row yet) and are best effort,+ // so they are let go in both modes; the ones already running finish with the rest.+ for (const { stage } of this.optInStages.values()) stage.close();if (!this.queueEnabled) {this.dispatchLimiter.close();for (const gate of this.degradedSessionGates.values()) gate.close();@@ -297,29 +306,56 @@ export class WebhookDeliveryService implements OnModuleInit, OnModuleDestroy {// An opt-in event fires per recipient (about 2,000 per message to a 1,000-member group): skip it// before the per-dispatch webhook read when no active webhook of the session names it.if (!(await this.sessionNamesOptInEvent(sessionId, event))) return;- if (this.optInPending >= Math.max(1, Math.floor(this.dispatchMaxQueued / 4))) {- if (this.optInDropped++ === 0) {- this.logger.warn(- `Dropping ${event} events: more are pending than a quarter of WEBHOOK_DISPATCH_MAX_QUEUED allows`,- { event, pending: this.optInPending, action: 'webhook_opt_in_dropping' },- );- }- return;+ let entry = this.optInStages.get(sessionId);+ if (!entry) {+ // A quarter of the pool runs at once (the limiter keeps at least one) and three quarters of the park bound+ // wait, at least one place.+ entry = {+ stage: new ConcurrencyLimiter(+ Math.floor(this.dispatchLimiter.maxCount / 4),+ Math.max(1, Math.floor((this.dispatchMaxQueued * 3) / 4)),+ ),+ dropped: 0,+ };+ this.optInStages.set(sessionId, entry);}- this.optInPending++;try {- await this.fanOut(sessionId, event, data);+ await entry.stage.run(() => this.fanOut(sessionId, event, data));+ } catch (error) {+ // Over the stage's bound, or still waiting when shutdown closed it; anything else is not ours to hide.+ if (isLimiterFull(error)) this.noteOptInDrop(sessionId, event, entry);+ else if (!isLimiterClosed(error)) throw error;} finally {- if (--this.optInPending === 0 && this.optInDropped > 0) {- this.logger.warn(`Dropped ${this.optInDropped} opt-in events while their allowance was full`, {- dropped: this.optInDropped,- action: 'webhook_opt_in_dropped',- });- this.optInDropped = 0;- }+ this.releaseOptInStage(sessionId, entry);}}+ /**+ * An opt-in event past its session's bound: counted with the other deliveries that were never sent, and+ * announced once per burst, when the session starts dropping. It leaves no row to replay, by design.+ */+ private noteOptInDrop(sessionId: string, event: string, entry: { dropped: number }): void {+ incrementWebhookDeliveryFailures();+ if (entry.dropped++ > 0) return;+ this.logger.warn(`Dropping ${event} events of session ${sessionId}: its queue of opt-in events is full`, {+ sessionId,+ event,+ action: 'webhook_opt_in_dropping',+ });+ }++ /** Forget a session's stage once it is idle, reporting how many events it turned away. */+ private releaseOptInStage(sessionId: string, entry: { stage: ConcurrencyLimiter; dropped: number }): void {+ if (entry.stage.activeCount > 0 || entry.stage.queuedCount > 0 || this.optInStages.get(sessionId) !== entry) return;+ this.optInStages.delete(sessionId);+ if (entry.dropped === 0) return;+ this.logger.warn(`Dropped ${entry.dropped} opt-in events of session ${sessionId} while its queue was full`, {+ sessionId,+ dropped: entry.dropped,+ action: 'webhook_opt_in_dropped',+ });+ }+private async fanOut(sessionId: string, event: string, data: Record<string, unknown>): Promise<void> {const webhooks = await this.loadActiveWebhooks(sessionId, event);const matchingWebhooks = this.filterMatchingWebhooks(webhooks, event, data);