Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(pacing): hold a bulk or product send until the engine answers

Yudhi Armyndharisb195acd3a051
diff --git a/src/modules/catalog/catalog.service.spec.ts b/src/modules/catalog/catalog.service.spec.ts
index 35ea87fb3..13ecb2277 100644
--- a/src/modules/catalog/catalog.service.spec.ts
+++ b/src/modules/catalog/catalog.service.spec.ts
@@ -128,7 +128,7 @@ describe('CatalogService', () => {
await expect(svc.sendProduct('s1', '[email protected]', 'prod-1', 'Back in stock!')).resolves.toBe(sent);
expect(order).toEqual(['pace', 'send']);
- expect(pacing.assertSendAllowed).toHaveBeenCalledWith('s1', '[email protected]');
+ expect(pacing.assertSendAllowed).toHaveBeenCalledWith('s1', '[email protected]', { untilSettled: true });
expect(sendProduct).toHaveBeenCalledWith('[email protected]', 'prod-1', 'Back in stock!');
});
@@ -216,7 +216,25 @@ describe('CatalogService', () => {
const { svc } = makeService(noEngine ? undefined : { sendProduct }, pacing, hookManager);
await expect(svc.sendProduct('s1', '[email protected]', 'prod-1')).rejects.toBeDefined();
- expect(release).toHaveBeenCalledTimes(released ? 1 : 0);
+ // Held otherwise for a window from the engine's answer, while the own-send echo may still land.
+ expect(release.mock.calls).toEqual([[!released]]);
+ });
+
+ // No row exists until the own-send echo, after the engine call, which a catalog query and an image
+ // fetch can stretch past the hold window: the admission is held until the engine returns.
+ it('holds its pacing admission until the engine call returns', async () => {
+ const release = jest.fn();
+ const pacing = { assertSendAllowed: jest.fn().mockResolvedValue(release) };
+ let settledDuringSend = -1;
+ const sendProduct = jest.fn().mockImplementation(() => {
+ settledDuringSend = release.mock.calls.length;
+ return Promise.resolve(sent);
+ });
+ const { svc } = makeService({ sendProduct }, pacing, passThroughHooks());
+
+ await expect(svc.sendProduct('s1', '[email protected]', 'prod-1')).resolves.toBe(sent);
+ expect(settledDuringSend).toBe(0);
+ expect(release.mock.calls).toEqual([[true]]);
});
it('propagates an engine failure from sendProduct', async () => {
diff --git a/src/modules/catalog/catalog.service.ts b/src/modules/catalog/catalog.service.ts
index 16e91055f..abf1323a6 100644
--- a/src/modules/catalog/catalog.service.ts
+++ b/src/modules/catalog/catalog.service.ts
@@ -50,10 +50,11 @@ export class CatalogService {
* API key's chat scope was checked against. No PENDING row is written up front. On Baileys the
* own-send echo (MessageProjector.handleOwnSendEcho) persists the OUTGOING row afterwards and fires
* `message:sent` and `message:persisted`, so the send is counted into the pacing daily cap once
- * that row lands.
+ * that row lands. Until then its pacing admission is held: for as long as the engine call runs, and for
+ * the hold window after it returns.
*/
async sendProduct(sessionId: string, chatId: string, productId: string, body?: string): Promise<MessageResult> {
- const release = await this.pacing.assertSendAllowed(sessionId, chatId);
+ const settle = await this.pacing.assertSendAllowed(sessionId, chatId, { untilSettled: true });
let engineAsked = false;
try {
const gated = await applySendingGate(
@@ -78,11 +79,14 @@ export class CatalogService {
() => new NotFoundException(`Session ${sessionId} not found or not connected`),
);
engineAsked = true;
- return await this.recordedSend(sessionId, () => engine.sendProduct(chatId, gatedProductId, gatedBody));
+ const result = await this.recordedSend(sessionId, () => engine.sendProduct(chatId, gatedProductId, gatedBody));
+ settle?.(true);
+ return result;
} catch (error) {
// No row is written here, so a send that provably never went out gives its pacing admission back.
- // One whose outcome is unknown stays held: its own-send echo may still write the row.
- if (!engineAsked || sentNothing(error)) release?.();
+ // One whose outcome is unknown stays held for a window from now: its own-send echo may still write
+ // the row.
+ settle?.(engineAsked && !sentNothing(error));
throw error;
}
}
diff --git a/src/modules/message/bulk-message.service.spec.ts b/src/modules/message/bulk-message.service.spec.ts
index 4a0f65eb0..4c99d1282 100644
--- a/src/modules/message/bulk-message.service.spec.ts
+++ b/src/modules/message/bulk-message.service.spec.ts
@@ -1044,6 +1044,7 @@ describe('BulkMessageService.processBatch', () => {
await runProcessBatch();
expect(release).toHaveBeenCalledTimes(1);
+ expect(release).not.toHaveBeenCalledWith(true);
});
it.each([
@@ -1056,7 +1057,7 @@ describe('BulkMessageService.processBatch', () => {
'the engine transport fails',
() => engine.sendTextMessage.mockRejectedValueOnce(new EngineTransportError('timeout')),
],
- ])('keeps the pacing admission held when %s', async (_label, arrange) => {
+ ])('keeps the pacing admission held for a window from the engine return when %s', async (_label, arrange) => {
const release = jest.fn();
pacing.assertSendAllowed.mockResolvedValue(release);
repo.findOne.mockResolvedValue(makeBatch(1));
@@ -1064,7 +1065,26 @@ describe('BulkMessageService.processBatch', () => {
await runProcessBatch();
- expect(release).not.toHaveBeenCalled();
+ expect(release.mock.calls).toEqual([[true]]);
+ });
+
+ // The item's row lands only after the engine call, which a media URL fetch or an upload can stretch past
+ // the hold window, so the admission is held until the engine returns rather than from the check.
+ it('holds the pacing admission until the engine call returns', async () => {
+ const release = jest.fn();
+ pacing.assertSendAllowed.mockResolvedValue(release);
+ repo.findOne.mockResolvedValue(makeBatch(1));
+ let settledDuringSend = -1;
+ engine.sendTextMessage.mockImplementationOnce(() => {
+ settledDuringSend = release.mock.calls.length;
+ return Promise.resolve({ id: 'wa1', timestamp: 111 });
+ });
+
+ await runProcessBatch();
+
+ expect(pacing.assertSendAllowed).toHaveBeenCalledWith('s1', expect.any(String), { untilSettled: true });
+ expect(settledDuringSend).toBe(0);
+ expect(release.mock.calls).toEqual([[true]]);
});
it('sends a bulk audio item with ptt as a voice note and persists type "voice"', async () => {
diff --git a/src/modules/message/bulk-message.service.ts b/src/modules/message/bulk-message.service.ts
index 6b4cc52a2..3ee626165 100644
--- a/src/modules/message/bulk-message.service.ts
+++ b/src/modules/message/bulk-message.service.ts
@@ -33,6 +33,7 @@ import {
countsTowardSendBreaker,
sentNothing,
SEND_PACING_LIMITED,
+ type SettleAdmission,
} from './send-pacing.service';
import { SessionOwnershipService } from '../session/session-ownership.service';
import { HookManager } from '../../core/hooks';
@@ -554,9 +555,10 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,
// decision (not a delivery failure) and skips message:failed — matching the single-send path,
// where a block is a 400 with no failure hook.
let blockedByPlugin = false;
- // The pacing admission's release. Bulk writes an item's row only after the engine accepts it, so an
- // item that fails first gives its admission back, or it would refuse the next item for the hold.
- let releaseAdmission: (() => void) | undefined;
+ // The pacing admission's settle. Bulk writes an item's row only after the engine accepts it, so the
+ // admission is held until the engine answers, and an item that fails first gives it back, or it
+ // would refuse the next item for the hold.
+ let settleAdmission: SettleAdmission | undefined;
let engineAsked = false;
try {
// Apply template variables
@@ -565,7 +567,7 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,
// Pacing runs BEFORE the moderation gate, matching MessageService: a send policy forbids is not
// offered to plugins at all. A refusal is a 429 that fails THIS item (honouring stopOnError),
// not the batch — the allowance may free up, and a batch killed outright could not resume.
- releaseAdmission = await this.pacing.assertSendAllowed(batch.sessionId, msg.chatId);
+ settleAdmission = await this.pacing.assertSendAllowed(batch.sessionId, msg.chatId, { untilSettled: true });
// Per-message moderation gate — the SAME message:sending hook single sends use, so a
// compliance/moderation plugin sees bulk traffic too (bulk previously bypassed it entirely).
@@ -628,10 +630,12 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,
if (countsTowardSendBreaker(engineError)) {
this.pacing.recordSendFailure(batch.sessionId);
}
- // A failure that may still have sent the message keeps its admission until the echo row lands.
- if (sentNothing(engineError)) releaseAdmission?.();
+ // A failure that may still have sent the message keeps its admission for a window from now, while
+ // the echo row lands.
+ settleAdmission?.(!sentNothing(engineError));
throw engineError;
}
+ settleAdmission?.(true);
this.pacing.recordSendSuccess(batch.sessionId);
result.status = BatchMessageStatus.SENT;
@@ -648,7 +652,7 @@ export class BulkMessageService implements OnModuleInit, OnApplicationBootstrap,
this.logger.debug(`Batch ${batch.batchId}: Sent message ${i + 1}/${batch.messages.length} to ${msg.chatId}`);
} catch (error) {
- if (!engineAsked) releaseAdmission?.();
+ if (!engineAsked) settleAdmission?.();
result.status = BatchMessageStatus.FAILED;
// Sanitize: an SSRF block names an internal address — never store/return/log it verbatim.
const sanitized = sanitizeBatchError(error);
diff --git a/src/modules/message/send-pacing-cold-query.spec.ts b/src/modules/message/send-pacing-cold-query.spec.ts
index 4cdb43f00..2b1cd34cd 100644
--- a/src/modules/message/send-pacing-cold-query.spec.ts
+++ b/src/modules/message/send-pacing-cold-query.spec.ts
@@ -472,6 +472,13 @@ describe('concurrent sends against a real database', () => {
: undefined,
} as unknown as ConfigService);
+ const insertRow = (id: string, chatId: string): Promise<unknown> =>
+ ds.query(
+ `INSERT INTO "messages" ("id","sessionId","chatId","from","to","type","direction","createdAt")
+ VALUES (?,?,?,'a','b','text','outgoing',?)`,
+ [id, 's1', chatId, new Date().toISOString()],
+ );
+
// Check, then persist the row the way a send path does; resolves true when the send went out.
const send = async (service: SendPacingService, chatId: string, i: number): Promise<boolean> => {
try {
@@ -479,11 +486,7 @@ describe('concurrent sends against a real database', () => {
} catch {
return false;
}
- await ds.query(
- `INSERT INTO "messages" ("id","sessionId","chatId","from","to","type","direction","createdAt")
- VALUES (?,?,?,'a','b','text','outgoing',?)`,
- [`m${i}`, 's1', chatId, new Date().toISOString()],
- );
+ await insertRow(`m${i}`, chatId);
return true;
};
@@ -666,6 +669,84 @@ describe('concurrent sends against a real database', () => {
});
});
+ it('keeps holding a send taken until settled while it is still running, then for a window from the settle', async () => {
+ const service = build([2], []);
+ expect(await send(service, '[email protected]', 1)).toBe(true);
+ // A bulk item whose engine call (a media URL fetch, an upload) outlasts the window.
+ const settle = await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });
+ jest.setSystemTime(NOW.getTime() + 11_000);
+
+ await expect(service.assertSendAllowed('s1', '[email protected]')).rejects.toMatchObject({
+ status: 429,
+ response: { code: SEND_PACING_LIMITED, retryAfterSeconds: 10 },
+ });
+
+ settle?.(true);
+ jest.setSystemTime(NOW.getTime() + 20_000);
+ await expect(service.assertSendAllowed('s1', '[email protected]')).rejects.toMatchObject({ status: 429 });
+ jest.setSystemTime(NOW.getTime() + 21_010);
+ await expect(service.assertSendAllowed('s1', '[email protected]')).resolves.toBeDefined();
+ });
+
+ it('keeps holding a cold send taken until settled while it is still running', async () => {
+ const service = build([10_000], [2]);
+ expect(await send(service, '[email protected]', 1)).toBe(true);
+ await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });
+ jest.setSystemTime(NOW.getTime() + 11_000);
+
+ await expect(service.assertSendAllowed('s1', '[email protected]')).rejects.toMatchObject({ status: 429 });
+ await expect(service.assertReachoutAllowed('s1', ['[email protected]'])).rejects.toMatchObject({ status: 429 });
+ });
+
+ it('counts a settled send for one still running that was admitted before its row landed', async () => {
+ const service = build([3], []);
+ expect(await send(service, '[email protected]', 1)).toBe(true);
+ const settleFirst = await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });
+ jest.setSystemTime(NOW.getTime() + 5_000);
+ // Admitted while the first is in flight, so the count it read cannot include the first's row.
+ await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });
+ jest.setSystemTime(NOW.getTime() + 30_000);
+ settleFirst?.(true);
+ await insertRow('m2', '[email protected]');
+ jest.setSystemTime(NOW.getTime() + 41_000);
+
+ await expect(service.assertSendAllowed('s1', '[email protected]')).rejects.toMatchObject({ status: 429 });
+ });
+
+ it('counts a settled cold send for one still running that was admitted before its row landed', async () => {
+ const service = build([10_000], [3]);
+ expect(await send(service, '[email protected]', 1)).toBe(true);
+ const settleFirst = await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });
+ jest.setSystemTime(NOW.getTime() + 5_000);
+ await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });
+ jest.setSystemTime(NOW.getTime() + 30_000);
+ settleFirst?.(true);
+ await insertRow('m2', '[email protected]');
+ jest.setSystemTime(NOW.getTime() + 41_000);
+
+ await expect(service.assertSendAllowed('s1', '[email protected]')).rejects.toMatchObject({ status: 429 });
+ });
+
+ it('hands back a send taken until settled at once when it is settled as never sent', async () => {
+ const service = build([2], []);
+ expect(await send(service, '[email protected]', 1)).toBe(true);
+ const settle = await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });
+ settle?.();
+
+ expect(await send(service, '[email protected]', 2)).toBe(true);
+ });
+
+ it('stops holding a send never settled after the safety bound', async () => {
+ const service = build([2], []);
+ expect(await send(service, '[email protected]', 1)).toBe(true);
+ await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });
+ jest.setSystemTime(NOW.getTime() + 5 * 60_000 - 1_000);
+ await expect(service.assertSendAllowed('s1', '[email protected]')).rejects.toMatchObject({ status: 429 });
+ jest.setSystemTime(NOW.getTime() + 5 * 60_000);
+
+ expect(await send(service, '[email protected]', 2)).toBe(true);
+ });
+
it('judges by the persisted rows alone once the hold has lapsed', async () => {
const service = build([5], []);
// A gated send that never writes a row (a plugin veto) is held only briefly.
diff --git a/src/modules/message/send-pacing.service.spec.ts b/src/modules/message/send-pacing.service.spec.ts
index d5a9cd5fd..a41af8a88 100644
--- a/src/modules/message/send-pacing.service.spec.ts
+++ b/src/modules/message/send-pacing.service.spec.ts
@@ -364,7 +364,7 @@ describe('send paths consult the governor', () => {
await expect(service.sendProduct('s1', '[email protected]', 'p1')).rejects.toBeInstanceOf(HttpException);
- expect(pacing.assertSendAllowed).toHaveBeenCalledWith('s1', '[email protected]');
+ expect(pacing.assertSendAllowed).toHaveBeenCalledWith('s1', '[email protected]', { untilSettled: true });
expect(hookManager.execute).not.toHaveBeenCalled();
expect(engine.sendProduct).not.toHaveBeenCalled();
});
@@ -423,7 +423,7 @@ describe('send paths consult the governor', () => {
await (service as unknown as { processBatch: (id: string) => Promise<void> }).processBatch('b1');
- expect(pacing.assertSendAllowed).toHaveBeenCalledWith('s1', '[email protected]');
+ expect(pacing.assertSendAllowed).toHaveBeenCalledWith('s1', '[email protected]', { untilSettled: true });
expect(hookManager.execute).not.toHaveBeenCalledWith('message:sending', expect.anything(), expect.anything());
expect(engine.sendTextMessage).not.toHaveBeenCalled();
});
diff --git a/src/modules/message/send-pacing.service.ts b/src/modules/message/send-pacing.service.ts
index b1701073b..d75767233 100644
--- a/src/modules/message/send-pacing.service.ts
+++ b/src/modules/message/send-pacing.service.ts
@@ -93,10 +93,13 @@ interface BreakerState {
* the caller after the check returns (after the plugin gate), so a burst of parallel requests would
* otherwise all read the same persisted count and all pass. Each cap is judged by the larger of the
* persisted count and, for every held admission, the count it read plus the admissions held from it
- * on: none of those rows can be in the count it read, so nothing is counted twice.
+ * on: none of those rows can be in the count it read, so nothing is counted twice. A send held until it
+ * settles is credited to the counts of the admissions taken after it when it settles, since they read
+ * before its row could land and would otherwise miss it once it stops being held.
*/
interface Admission {
- at: number;
+ /** Epoch ms the admission stops being held, so only its row counts from then on. */
+ until: number;
sentToday: number;
/** Set for a cold send; the key is folded the way the cold count folds dialects. */
cold: { key: string; coldToday: number } | null;
@@ -109,13 +112,30 @@ interface AdmissionHold {
}
/**
- * How long each admitted send is held against the caps, from its own admission, before only its row
- * counts. It only has to outlast the gap between the check and the row insert. A send that fails before
- * writing a row hands its admission back (see assertSendAllowed), so only one that may still have gone
- * out is over-counted, and for at most this long.
+ * How long each admitted send is held against the caps before only its row counts: from its own
+ * admission, or for one taken `untilSettled`, from the moment its caller settles it. It only has to
+ * outlast the gap between that moment and the row insert. A send that fails before writing a row hands
+ * its admission back (see assertSendAllowed), so only one that may still have gone out is over-counted,
+ * and for at most this long.
*/
const ADMISSION_WINDOW_MS = 10_000;
+/**
+ * Safety bound on an admission taken `untilSettled` that its caller never settles. Far longer than an
+ * engine send normally runs (a media URL fetch is bounded by MEDIA_DOWNLOAD_TIMEOUT_MS, 30 s by default),
+ * so it only matters if a caller loses track of one, which then holds the caps for this long instead of
+ * for the rest of the day. A send still running past it stops being held.
+ */
+const UNSETTLED_ADMISSION_MAX_MS = 5 * 60_000;
+
+/**
+ * Settles the admission assertSendAllowed held. Called with nothing, for a send that provably never went
+ * out, it hands the admission back at once. Called with `true`, once the engine returned or failed with an
+ * unknown outcome, it holds the admission for ADMISSION_WINDOW_MS from now, while the row or the own-send
+ * echo lands.
+ */
+export type SettleAdmission = (mayHaveSent?: boolean) => void;
+
/** Refusals suppressed since the last audited one, per session. */
interface RefusalSample {
count: number;
@@ -161,7 +181,9 @@ const MAX_REFUSAL_KEYS = 1000;
* (MessageProjector.handleOwnSendEcho) shortly after the send returns. Deliberate, and documented in
* .env.example and docs/06 so the number an operator reads is the number they get.
* Because the row lands only after the check returns, sends admitted in the last few seconds are
- * also held in memory (see Admission), or a parallel burst would pass against one stale count.
+ * also held in memory (see Admission), or a parallel burst would pass against one stale count. A bulk
+ * item and a product send write their row only after the engine call, which can outlast that window
+ * (a media URL fetch, an upload), so they are held until they settle (see `untilSettled`).
* The breaker, by contrast, is in memory on purpose: it describes live conditions, and a restart
* clearing it is the correct behaviour.
*/
@@ -205,15 +227,20 @@ export class SendPacingService {
* `hold: false` judges the send without holding it in the admission window, for a gated send that
* never writes a row (an edit): holding it would charge the caps for a message that is never sent.
*
- * Returns the release for the admission it held, or undefined when it held none. A caller whose send
- * fails before writing a row, and before WhatsApp may have taken it, calls the release so that send
- * stops counting against the caps at once instead of refusing the next one.
+ * `untilSettled` holds the admission for as long as the send runs, for a caller whose row lands only
+ * after the engine call (a bulk item, a product send): the window starts when the caller settles it
+ * with `true`, bounded by UNSETTLED_ADMISSION_MAX_MS if it never does. Without it the window starts
+ * now, for a caller that writes its PENDING row before asking the engine.
+ *
+ * Returns the settle for the admission it held, or undefined when it held none. A caller whose send
+ * fails before writing a row, and before WhatsApp may have taken it, calls it with nothing so that
+ * send stops counting against the caps at once instead of refusing the next one.
*/
async assertSendAllowed(
sessionId: string,
chatId?: string,
- opts: { hold?: boolean } = {},
- ): Promise<(() => void) | undefined> {
+ opts: { hold?: boolean; untilSettled?: boolean } = {},
+ ): Promise<SettleAdmission | undefined> {
const config = resolveSendPacingConfig(this.configService);
if (!config.enabled) return;
@@ -246,17 +273,29 @@ export class SendPacingService {
}
if (opts.hold !== false) {
const admission: Admission = {
- at: Date.now(),
+ until: Date.now() + (opts.untilSettled ? UNSETTLED_ADMISSION_MAX_MS : ADMISSION_WINDOW_MS),
sentToday,
cold: cold ? { key, coldToday: cold.coldToday } : null,
};
const hold = this.holds.get(sessionId);
if (hold?.dayStartMs === dayStart.getTime()) hold.admissions.push(admission);
else this.holds.set(sessionId, { dayStartMs: dayStart.getTime(), admissions: [admission] });
- return () => {
+ return (mayHaveSent = false) => {
const admissions = this.holds.get(sessionId)?.admissions;
const index = admissions?.indexOf(admission) ?? -1;
- if (admissions && index !== -1) admissions.splice(index, 1);
+ if (!mayHaveSent) {
+ if (admissions && index !== -1) admissions.splice(index, 1);
+ return;
+ }
+ admission.until = Date.now() + ADMISSION_WINDOW_MS;
+ if (!admissions || index === -1) return;
+ // Every admission held after this one was taken while it was in flight, so the counts it read
+ // cannot include this send's coming row; credit it there, or once this one lapses a send still
+ // running would be judged without it.
+ for (const later of admissions.slice(index + 1)) {
+ later.sentToday += 1;
+ if (admission.cold && later.cold && later.cold.key !== admission.cold.key) later.cold.coldToday += 1;
+ }
};
}
}
@@ -281,7 +320,7 @@ export class SendPacingService {
const hold = this.holds.get(sessionId);
if (!hold) return [];
const now = Date.now();
- const live = hold.admissions.findIndex(a => now - a.at < ADMISSION_WINDOW_MS);
+ const live = hold.admissions.findIndex(a => a.until > now);
if (hold.dayStartMs === dayStart.getTime() && live !== -1) {
hold.admissions.splice(0, live);
return hold.admissions;
@@ -711,12 +750,14 @@ function startOfUtcDay(at: Date): Date {
}
/**
- * Seconds until every one of these held admissions has lapsed: when the persisted count alone is under
- * the cap, the refusal lifts then, not at the next UTC day.
+ * Seconds until every one of these held admissions could have lapsed: when the persisted count alone is
+ * under the cap, the refusal lifts then, not at the next UTC day. One still unsettled lapses a window
+ * after its settle at the earliest, so it is hinted as if it settled now.
*/
function secondsUntilLapsed(held: Admission[]): number {
- const newest = held.at(-1)?.at ?? Date.now();
- return Math.max(1, Math.ceil((newest + ADMISSION_WINDOW_MS - Date.now()) / 1000));
+ const now = Date.now();
+ const last = held.reduce((max, a) => Math.max(max, Math.min(a.until, now + ADMISSION_WINDOW_MS)), now);
+ return Math.max(1, Math.ceil((last - now) / 1000));
}
function secondsUntilNextUtcDay(): number {