Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(pacing): hold a bulk or product send until the engine answers
Yudhi Armyndharisb195acd3a051
diff --git a/src/modules/catalog/catalog.service.spec.ts b/src/modules/catalog/catalog.service.spec.tsindex 35ea87fb3..13ecb2277 100644--- a/src/modules/catalog/catalog.service.spec.ts+++ b/src/modules/catalog/catalog.service.spec.ts@@ -128,7 +128,7 @@ describe('CatalogService', () => {await expect(svc.sendProduct('s1', '[email protected]', 'prod-1', 'Back in stock!')).resolves.toBe(sent);expect(order).toEqual(['pace', 'send']);- expect(pacing.assertSendAllowed).toHaveBeenCalledWith('s1', '[email protected]');+ expect(pacing.assertSendAllowed).toHaveBeenCalledWith('s1', '[email protected]', { untilSettled: true });expect(sendProduct).toHaveBeenCalledWith('[email protected]', 'prod-1', 'Back in stock!');});@@ -216,7 +216,25 @@ describe('CatalogService', () => {const { svc } = makeService(noEngine ? undefined : { sendProduct }, pacing, hookManager);await expect(svc.sendProduct('s1', '[email protected]', 'prod-1')).rejects.toBeDefined();- expect(release).toHaveBeenCalledTimes(released ? 1 : 0);+ // Held otherwise for a window from the engine's answer, while the own-send echo may still land.+ expect(release.mock.calls).toEqual([[!released]]);+ });++ // No row exists until the own-send echo, after the engine call, which a catalog query and an image+ // fetch can stretch past the hold window: the admission is held until the engine returns.+ it('holds its pacing admission until the engine call returns', async () => {+ const release = jest.fn();+ const pacing = { assertSendAllowed: jest.fn().mockResolvedValue(release) };+ let settledDuringSend = -1;+ const sendProduct = jest.fn().mockImplementation(() => {+ settledDuringSend = release.mock.calls.length;+ return Promise.resolve(sent);+ });+ const { svc } = makeService({ sendProduct }, pacing, passThroughHooks());++ await expect(svc.sendProduct('s1', '[email protected]', 'prod-1')).resolves.toBe(sent);+ expect(settledDuringSend).toBe(0);+ expect(release.mock.calls).toEqual([[true]]);});it('propagates an engine failure from sendProduct', async () => {diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.tsindex 16e91055f..abf1323a6 100644--- a/src/modules/catalog/catalog.service.ts+++ b/src/modules/catalog/catalog.service.ts@@ -50,10 +50,11 @@ export class CatalogService {* API key's chat scope was checked against. No PENDING row is written up front. On Baileys the* own-send echo (MessageProjector.handleOwnSendEcho) persists the OUTGOING row afterwards and fires* `message:sent` and `message:persisted`, so the send is counted into the pacing daily cap once- * that row lands.+ * that row lands. Until then its pacing admission is held: for as long as the engine call runs, and for+ * the hold window after it returns.*/async sendProduct(sessionId: string, chatId: string, productId: string, body?: string): Promise<MessageResult> {- const release = await this.pacing.assertSendAllowed(sessionId, chatId);+ const settle = await this.pacing.assertSendAllowed(sessionId, chatId, { untilSettled: true });let engineAsked = false;try {const gated = await applySendingGate(@@ -78,11 +79,14 @@ export class CatalogService {() => new NotFoundException(`Session ${sessionId} not found or not connected`),);engineAsked = true;- return await this.recordedSend(sessionId, () => engine.sendProduct(chatId, gatedProductId, gatedBody));+ const result = await this.recordedSend(sessionId, () => engine.sendProduct(chatId, gatedProductId, gatedBody));+ settle?.(true);+ return result;} catch (error) {// No row is written here, so a send that provably never went out gives its pacing admission back.- // One whose outcome is unknown stays held: its own-send echo may still write the row.- if (!engineAsked || sentNothing(error)) release?.();+ // One whose outcome is unknown stays held for a window from now: its own-send echo may still write+ // the row.+ settle?.(engineAsked && !sentNothing(error));throw error;}}diff --git a/src/modules/message/bulk-message.service.spec.ts b/src/modules/message/bulk-message.service.spec.tsindex 4a0f65eb0..4c99d1282 100644--- a/src/modules/message/bulk-message.service.spec.ts+++ b/src/modules/message/bulk-message.service.spec.ts@@ -1044,6 +1044,7 @@ describe('BulkMessageService.processBatch', () => {await runProcessBatch();expect(release).toHaveBeenCalledTimes(1);+ expect(release).not.toHaveBeenCalledWith(true);});it.each([@@ -1056,7 +1057,7 @@ describe('BulkMessageService.processBatch', () => {'the engine transport fails',() => engine.sendTextMessage.mockRejectedValueOnce(new EngineTransportError('timeout')),],- ])('keeps the pacing admission held when %s', async (_label, arrange) => {+ ])('keeps the pacing admission held for a window from the engine return when %s', async (_label, arrange) => {const release = jest.fn();pacing.assertSendAllowed.mockResolvedValue(release);repo.findOne.mockResolvedValue(makeBatch(1));@@ -1064,7 +1065,26 @@ describe('BulkMessageService.processBatch', () => {await runProcessBatch();- expect(release).not.toHaveBeenCalled();+ expect(release.mock.calls).toEqual([[true]]);+ });++ // The item's row lands only after the engine call, which a media URL fetch or an upload can stretch past+ // the hold window, so the admission is held until the engine returns rather than from the check.+ it('holds the pacing admission until the engine call returns', async () => {+ const release = jest.fn();+ pacing.assertSendAllowed.mockResolvedValue(release);+ repo.findOne.mockResolvedValue(makeBatch(1));+ let settledDuringSend = -1;+ engine.sendTextMessage.mockImplementationOnce(() => {+ settledDuringSend = release.mock.calls.length;+ return Promise.resolve({ id: 'wa1', timestamp: 111 });+ });++ await runProcessBatch();++ expect(pacing.assertSendAllowed).toHaveBeenCalledWith('s1', expect.any(String), { untilSettled: true });+ expect(settledDuringSend).toBe(0);+ expect(release.mock.calls).toEqual([[true]]);});it('sends a bulk audio item with ptt as a voice note and persists type "voice"', async () => {diff --git a/src/modules/message/bulk-message.service.ts b/src/modules/message/bulk-message.service.tsindex 6b4cc52a2..3ee626165 100644--- a/src/modules/message/bulk-message.service.ts+++ b/src/modules/message/bulk-message.service.ts@@ -33,6 +33,7 @@ import {countsTowardSendBreaker,sentNothing,SEND_PACING_LIMITED,+ type SettleAdmission,} from './send-pacing.service';import { SessionOwnershipService } from '../session/session-ownership.service';import { HookManager } from '../../core/hooks';@@ -554,9 +555,10 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,// decision (not a delivery failure) and skips message:failed — matching the single-send path,// where a block is a 400 with no failure hook.let blockedByPlugin = false;- // The pacing admission's release. Bulk writes an item's row only after the engine accepts it, so an- // item that fails first gives its admission back, or it would refuse the next item for the hold.- let releaseAdmission: (() => void) | undefined;+ // The pacing admission's settle. Bulk writes an item's row only after the engine accepts it, so the+ // admission is held until the engine answers, and an item that fails first gives it back, or it+ // would refuse the next item for the hold.+ let settleAdmission: SettleAdmission | undefined;let engineAsked = false;try {// Apply template variables@@ -565,7 +567,7 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,// Pacing runs BEFORE the moderation gate, matching MessageService: a send policy forbids is not// offered to plugins at all. A refusal is a 429 that fails THIS item (honouring stopOnError),// not the batch — the allowance may free up, and a batch killed outright could not resume.- releaseAdmission = await this.pacing.assertSendAllowed(batch.sessionId, msg.chatId);+ settleAdmission = await this.pacing.assertSendAllowed(batch.sessionId, msg.chatId, { untilSettled: true });// Per-message moderation gate — the SAME message:sending hook single sends use, so a// compliance/moderation plugin sees bulk traffic too (bulk previously bypassed it entirely).@@ -628,10 +630,12 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,if (countsTowardSendBreaker(engineError)) {this.pacing.recordSendFailure(batch.sessionId);}- // A failure that may still have sent the message keeps its admission until the echo row lands.- if (sentNothing(engineError)) releaseAdmission?.();+ // A failure that may still have sent the message keeps its admission for a window from now, while+ // the echo row lands.+ settleAdmission?.(!sentNothing(engineError));throw engineError;}+ settleAdmission?.(true);this.pacing.recordSendSuccess(batch.sessionId);result.status = BatchMessageStatus.SENT;@@ -648,7 +652,7 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,this.logger.debug(`Batch ${batch.batchId}: Sent message ${i + 1}/${batch.messages.length} to ${msg.chatId}`);} catch (error) {- if (!engineAsked) releaseAdmission?.();+ if (!engineAsked) settleAdmission?.();result.status = BatchMessageStatus.FAILED;// Sanitize: an SSRF block names an internal address — never store/return/log it verbatim.const sanitized = sanitizeBatchError(error);diff --git a/src/modules/message/send-pacing-cold-query.spec.ts b/src/modules/message/send-pacing-cold-query.spec.tsindex 4cdb43f00..2b1cd34cd 100644--- a/src/modules/message/send-pacing-cold-query.spec.ts+++ b/src/modules/message/send-pacing-cold-query.spec.ts@@ -472,6 +472,13 @@ describe('concurrent sends against a real database', () => {: undefined,} as unknown as ConfigService);+ const insertRow = (id: string, chatId: string): Promise<unknown> =>+ ds.query(+ `INSERT INTO "messages" ("id","sessionId","chatId","from","to","type","direction","createdAt")+ VALUES (?,?,?,'a','b','text','outgoing',?)`,+ [id, 's1', chatId, new Date().toISOString()],+ );+// Check, then persist the row the way a send path does; resolves true when the send went out.const send = async (service: SendPacingService, chatId: string, i: number): Promise<boolean> => {try {@@ -479,11 +486,7 @@ describe('concurrent sends against a real database', () => {} catch {return false;}- await ds.query(- `INSERT INTO "messages" ("id","sessionId","chatId","from","to","type","direction","createdAt")- VALUES (?,?,?,'a','b','text','outgoing',?)`,- [`m${i}`, 's1', chatId, new Date().toISOString()],- );+ await insertRow(`m${i}`, chatId);return true;};@@ -666,6 +669,84 @@ describe('concurrent sends against a real database', () => {});});+ it('keeps holding a send taken until settled while it is still running, then for a window from the settle', async () => {+ const service = build([2], []);+ expect(await send(service, '[email protected]', 1)).toBe(true);+ // A bulk item whose engine call (a media URL fetch, an upload) outlasts the window.+ const settle = await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });+ jest.setSystemTime(NOW.getTime() + 11_000);++ await expect(service.assertSendAllowed('s1', '[email protected]')).rejects.toMatchObject({+ status: 429,+ response: { code: SEND_PACING_LIMITED, retryAfterSeconds: 10 },+ });++ settle?.(true);+ jest.setSystemTime(NOW.getTime() + 20_000);+ await expect(service.assertSendAllowed('s1', '[email protected]')).rejects.toMatchObject({ status: 429 });+ jest.setSystemTime(NOW.getTime() + 21_010);+ await expect(service.assertSendAllowed('s1', '[email protected]')).resolves.toBeDefined();+ });++ it('keeps holding a cold send taken until settled while it is still running', async () => {+ const service = build([10_000], [2]);+ expect(await send(service, '[email protected]', 1)).toBe(true);+ await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });+ jest.setSystemTime(NOW.getTime() + 11_000);++ await expect(service.assertSendAllowed('s1', '[email protected]')).rejects.toMatchObject({ status: 429 });+ await expect(service.assertReachoutAllowed('s1', ['[email protected]'])).rejects.toMatchObject({ status: 429 });+ });++ it('counts a settled send for one still running that was admitted before its row landed', async () => {+ const service = build([3], []);+ expect(await send(service, '[email protected]', 1)).toBe(true);+ const settleFirst = await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });+ jest.setSystemTime(NOW.getTime() + 5_000);+ // Admitted while the first is in flight, so the count it read cannot include the first's row.+ await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });+ jest.setSystemTime(NOW.getTime() + 30_000);+ settleFirst?.(true);+ await insertRow('m2', '[email protected]');+ jest.setSystemTime(NOW.getTime() + 41_000);++ await expect(service.assertSendAllowed('s1', '[email protected]')).rejects.toMatchObject({ status: 429 });+ });++ it('counts a settled cold send for one still running that was admitted before its row landed', async () => {+ const service = build([10_000], [3]);+ expect(await send(service, '[email protected]', 1)).toBe(true);+ const settleFirst = await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });+ jest.setSystemTime(NOW.getTime() + 5_000);+ await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });+ jest.setSystemTime(NOW.getTime() + 30_000);+ settleFirst?.(true);+ await insertRow('m2', '[email protected]');+ jest.setSystemTime(NOW.getTime() + 41_000);++ await expect(service.assertSendAllowed('s1', '[email protected]')).rejects.toMatchObject({ status: 429 });+ });++ it('hands back a send taken until settled at once when it is settled as never sent', async () => {+ const service = build([2], []);+ expect(await send(service, '[email protected]', 1)).toBe(true);+ const settle = await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });+ settle?.();++ expect(await send(service, '[email protected]', 2)).toBe(true);+ });++ it('stops holding a send never settled after the safety bound', async () => {+ const service = build([2], []);+ expect(await send(service, '[email protected]', 1)).toBe(true);+ await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });+ jest.setSystemTime(NOW.getTime() + 5 * 60_000 - 1_000);+ await expect(service.assertSendAllowed('s1', '[email protected]')).rejects.toMatchObject({ status: 429 });+ jest.setSystemTime(NOW.getTime() + 5 * 60_000);++ expect(await send(service, '[email protected]', 2)).toBe(true);+ });+it('judges by the persisted rows alone once the hold has lapsed', async () => {const service = build([5], []);// A gated send that never writes a row (a plugin veto) is held only briefly.diff --git a/src/modules/message/send-pacing.service.spec.ts b/src/modules/message/send-pacing.service.spec.tsindex d5a9cd5fd..a41af8a88 100644--- a/src/modules/message/send-pacing.service.spec.ts+++ b/src/modules/message/send-pacing.service.spec.ts@@ -364,7 +364,7 @@ describe('send paths consult the governor', () => {await expect(service.sendProduct('s1', '[email protected]', 'p1')).rejects.toBeInstanceOf(HttpException);- expect(pacing.assertSendAllowed).toHaveBeenCalledWith('s1', '[email protected]');+ expect(pacing.assertSendAllowed).toHaveBeenCalledWith('s1', '[email protected]', { untilSettled: true });expect(hookManager.execute).not.toHaveBeenCalled();expect(engine.sendProduct).not.toHaveBeenCalled();});@@ -423,7 +423,7 @@ describe('send paths consult the governor', () => {await (service as unknown as { processBatch: (id: string) => Promise<void> }).processBatch('b1');- expect(pacing.assertSendAllowed).toHaveBeenCalledWith('s1', '[email protected]');+ expect(pacing.assertSendAllowed).toHaveBeenCalledWith('s1', '[email protected]', { untilSettled: true });expect(hookManager.execute).not.toHaveBeenCalledWith('message:sending', expect.anything(), expect.anything());expect(engine.sendTextMessage).not.toHaveBeenCalled();});diff --git a/src/modules/message/send-pacing.service.ts b/src/modules/message/send-pacing.service.tsindex b1701073b..d75767233 100644--- a/src/modules/message/send-pacing.service.ts+++ b/src/modules/message/send-pacing.service.ts@@ -93,10 +93,13 @@ interface BreakerState {* the caller after the check returns (after the plugin gate), so a burst of parallel requests would* otherwise all read the same persisted count and all pass. Each cap is judged by the larger of the* persisted count and, for every held admission, the count it read plus the admissions held from it- * on: none of those rows can be in the count it read, so nothing is counted twice.+ * on: none of those rows can be in the count it read, so nothing is counted twice. A send held until it+ * settles is credited to the counts of the admissions taken after it when it settles, since they read+ * before its row could land and would otherwise miss it once it stops being held.*/interface Admission {- at: number;+ /** Epoch ms the admission stops being held, so only its row counts from then on. */+ until: number;sentToday: number;/** Set for a cold send; the key is folded the way the cold count folds dialects. */cold: { key: string; coldToday: number } | null;@@ -109,13 +112,30 @@ interface AdmissionHold {}/**- * How long each admitted send is held against the caps, from its own admission, before only its row- * counts. It only has to outlast the gap between the check and the row insert. A send that fails before- * writing a row hands its admission back (see assertSendAllowed), so only one that may still have gone- * out is over-counted, and for at most this long.+ * How long each admitted send is held against the caps before only its row counts: from its own+ * admission, or for one taken `untilSettled`, from the moment its caller settles it. It only has to+ * outlast the gap between that moment and the row insert. A send that fails before writing a row hands+ * its admission back (see assertSendAllowed), so only one that may still have gone out is over-counted,+ * and for at most this long.*/const ADMISSION_WINDOW_MS = 10_000;+/**+ * Safety bound on an admission taken `untilSettled` that its caller never settles. Far longer than an+ * engine send normally runs (a media URL fetch is bounded by MEDIA_DOWNLOAD_TIMEOUT_MS, 30 s by default),+ * so it only matters if a caller loses track of one, which then holds the caps for this long instead of+ * for the rest of the day. A send still running past it stops being held.+ */+const UNSETTLED_ADMISSION_MAX_MS = 5 * 60_000;++/**+ * Settles the admission assertSendAllowed held. Called with nothing, for a send that provably never went+ * out, it hands the admission back at once. Called with `true`, once the engine returned or failed with an+ * unknown outcome, it holds the admission for ADMISSION_WINDOW_MS from now, while the row or the own-send+ * echo lands.+ */+export type SettleAdmission = (mayHaveSent?: boolean) => void;+/** Refusals suppressed since the last audited one, per session. */interface RefusalSample {count: number;@@ -161,7 +181,9 @@ const MAX_REFUSAL_KEYS = 1000;* (MessageProjector.handleOwnSendEcho) shortly after the send returns. Deliberate, and documented in* .env.example and docs/06 so the number an operator reads is the number they get.* Because the row lands only after the check returns, sends admitted in the last few seconds are- * also held in memory (see Admission), or a parallel burst would pass against one stale count.+ * also held in memory (see Admission), or a parallel burst would pass against one stale count. A bulk+ * item and a product send write their row only after the engine call, which can outlast that window+ * (a media URL fetch, an upload), so they are held until they settle (see `untilSettled`).* The breaker, by contrast, is in memory on purpose: it describes live conditions, and a restart* clearing it is the correct behaviour.*/@@ -205,15 +227,20 @@ export class SendPacingService {* `hold: false` judges the send without holding it in the admission window, for a gated send that* never writes a row (an edit): holding it would charge the caps for a message that is never sent.*- * Returns the release for the admission it held, or undefined when it held none. A caller whose send- * fails before writing a row, and before WhatsApp may have taken it, calls the release so that send- * stops counting against the caps at once instead of refusing the next one.+ * `untilSettled` holds the admission for as long as the send runs, for a caller whose row lands only+ * after the engine call (a bulk item, a product send): the window starts when the caller settles it+ * with `true`, bounded by UNSETTLED_ADMISSION_MAX_MS if it never does. Without it the window starts+ * now, for a caller that writes its PENDING row before asking the engine.+ *+ * Returns the settle for the admission it held, or undefined when it held none. A caller whose send+ * fails before writing a row, and before WhatsApp may have taken it, calls it with nothing so that+ * send stops counting against the caps at once instead of refusing the next one.*/async assertSendAllowed(sessionId: string,chatId?: string,- opts: { hold?: boolean } = {},- ): Promise<(() => void) | undefined> {+ opts: { hold?: boolean; untilSettled?: boolean } = {},+ ): Promise<SettleAdmission | undefined> {const config = resolveSendPacingConfig(this.configService);if (!config.enabled) return;@@ -246,17 +273,29 @@ export class SendPacingService {}if (opts.hold !== false) {const admission: Admission = {- at: Date.now(),+ until: Date.now() + (opts.untilSettled ? UNSETTLED_ADMISSION_MAX_MS : ADMISSION_WINDOW_MS),sentToday,cold: cold ? { key, coldToday: cold.coldToday } : null,};const hold = this.holds.get(sessionId);if (hold?.dayStartMs === dayStart.getTime()) hold.admissions.push(admission);else this.holds.set(sessionId, { dayStartMs: dayStart.getTime(), admissions: [admission] });- return () => {+ return (mayHaveSent = false) => {const admissions = this.holds.get(sessionId)?.admissions;const index = admissions?.indexOf(admission) ?? -1;- if (admissions && index !== -1) admissions.splice(index, 1);+ if (!mayHaveSent) {+ if (admissions && index !== -1) admissions.splice(index, 1);+ return;+ }+ admission.until = Date.now() + ADMISSION_WINDOW_MS;+ if (!admissions || index === -1) return;+ // Every admission held after this one was taken while it was in flight, so the counts it read+ // cannot include this send's coming row; credit it there, or once this one lapses a send still+ // running would be judged without it.+ for (const later of admissions.slice(index + 1)) {+ later.sentToday += 1;+ if (admission.cold && later.cold && later.cold.key !== admission.cold.key) later.cold.coldToday += 1;+ }};}}@@ -281,7 +320,7 @@ export class SendPacingService {const hold = this.holds.get(sessionId);if (!hold) return [];const now = Date.now();- const live = hold.admissions.findIndex(a => now - a.at < ADMISSION_WINDOW_MS);+ const live = hold.admissions.findIndex(a => a.until > now);if (hold.dayStartMs === dayStart.getTime() && live !== -1) {hold.admissions.splice(0, live);return hold.admissions;@@ -711,12 +750,14 @@ function startOfUtcDay(at: Date): Date {}/**- * Seconds until every one of these held admissions has lapsed: when the persisted count alone is under- * the cap, the refusal lifts then, not at the next UTC day.+ * Seconds until every one of these held admissions could have lapsed: when the persisted count alone is+ * under the cap, the refusal lifts then, not at the next UTC day. One still unsettled lapses a window+ * after its settle at the earliest, so it is hinted as if it settled now.*/function secondsUntilLapsed(held: Admission[]): number {- const newest = held.at(-1)?.at ?? Date.now();- return Math.max(1, Math.ceil((newest + ADMISSION_WINDOW_MS - Date.now()) / 1000));+ const now = Date.now();+ const last = held.reduce((max, a) => Math.max(max, Math.min(a.until, now + ADMISSION_WINDOW_MS)), now);+ return Math.max(1, Math.ceil((last - now) / 1000));}function secondsUntilNextUtcDay(): number {