Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(pacing): stop warm sends from holding a cold send's count

Yudhi Armyndharis005412ce5fda
diff --git a/src/modules/message/send-pacing-cold-query.spec.ts b/src/modules/message/send-pacing-cold-query.spec.ts
index d79a90807..56bd45f8d 100644
--- a/src/modules/message/send-pacing-cold-query.spec.ts
+++ b/src/modules/message/send-pacing-cold-query.spec.ts
@@ -667,6 +667,79 @@ describe('concurrent sends against a real database', () => {
});
});
+ describe('a cold send with no row in a busy session', () => {
+ const seedKnown = (): Promise<unknown> =>
+ ds.query(
+ `INSERT INTO "messages" ("id","sessionId","chatId","from","to","type","direction","createdAt")
+ VALUES ('seed','s1','[email protected]','a','b','text','incoming',?)`,
+ [new Date(NOW.getTime() - 86_400_000).toISOString()],
+ );
+
+ // A cold bulk item with an unknown outcome (it never writes a row), a cold send whose row lands,
+ // then a warm send every 3 s for 30 s.
+ const busySession = async (service: SendPacingService): Promise<void> => {
+ await seedKnown();
+ const settle = await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });
+ settle?.(true);
+ expect(await send(service, '[email protected]', 1)).toBe(true);
+ for (let t = 1; t <= 10; t++) {
+ jest.setSystemTime(NOW.getTime() + t * 3_000);
+ expect(await send(service, '[email protected]', 10 + t)).toBe(true);
+ }
+ };
+
+ it('stops counting it once its own window ends, however busy the session', async () => {
+ const service = build([10_000], [2]);
+ await busySession(service);
+
+ // One new chat (y) of two reached.
+ expect(await send(service, '[email protected]', 99)).toBe(true);
+ });
+
+ it('stops charging it to a group add once its own window ends, however busy the session', async () => {
+ const service = build([10_000], [2]);
+ await busySession(service);
+
+ await expect(service.assertReachoutAllowed('s1', ['[email protected]'])).resolves.toMatchObject({ coldCount: 1 });
+ });
+
+ it('judges a second send to that chat against the cap once the first stopped counting', async () => {
+ const service = build([10_000], [2]);
+ await busySession(service);
+ expect(await send(service, '[email protected]', 99)).toBe(true);
+
+ // y and z reached; x would be a third new chat.
+ await expect(service.assertSendAllowed('s1', '[email protected]')).rejects.toMatchObject({ status: 429 });
+ });
+ });
+
+ it.each([
+ ['send', (service: SendPacingService) => service.assertSendAllowed('s1', '[email protected]')],
+ ['group add', (service: SendPacingService) => service.assertReachoutAllowed('s1', ['[email protected]'])],
+ ])('answers a cold %s refused by held sends with a hint a retry then passes', async (_, attempt) => {
+ const service = build([10_000], [1]);
+ await ds.query(
+ `INSERT INTO "messages" ("id","sessionId","chatId","from","to","type","direction","createdAt")
+ VALUES ('seed','s1','[email protected]','a','b','text','incoming',?)`,
+ [new Date(NOW.getTime() - 86_400_000).toISOString()],
+ );
+ // A bulk item to a known chat, in flight until +5 s, and a cold send admitted meanwhile whose row never lands.
+ const settleKnown = await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });
+ jest.setSystemTime(NOW.getTime() + 1_000);
+ await service.assertSendAllowed('s1', '[email protected]');
+ jest.setSystemTime(NOW.getTime() + 5_000);
+ settleKnown?.(true);
+ await insertRow('m1', '[email protected]');
+ jest.setSystemTime(NOW.getTime() + 6_000);
+
+ const refusal = (await attempt(service).catch((error: HttpException) => error.getResponse())) as {
+ retryAfterSeconds: number;
+ };
+ jest.setSystemTime(NOW.getTime() + 6_000 + refusal.retryAfterSeconds * 1_000);
+
+ await expect(attempt(service)).resolves.toBeDefined();
+ });
+
it('keeps holding a send taken until settled while it is still running, then for a window from the settle', async () => {
const service = build([2], []);
expect(await send(service, '[email protected]', 1)).toBe(true);
@@ -816,7 +889,7 @@ describe('concurrent sends against a real database', () => {
await expect(service.assertSendAllowed('s1', '[email protected]')).rejects.toMatchObject({ status: 429 });
});
- it('hints a settled cold send as held until the send admitted while it was in flight lapses', async () => {
+ it('stops counting a settled cold send once its own window ends, though a warm send admitted meanwhile is held', async () => {
const service = build([10_000], [1]);
await ds.query(
`INSERT INTO "messages" ("id","sessionId","chatId","from","to","type","direction","createdAt")
@@ -828,11 +901,14 @@ describe('concurrent sends against a real database', () => {
settleCold?.(true);
jest.setSystemTime(NOW.getTime() + 5_000);
settleKnown?.(true);
- jest.setSystemTime(NOW.getTime() + 12_000);
-
+ jest.setSystemTime(NOW.getTime() + 9_000);
await expect(service.assertSendAllowed('s1', '[email protected]')).rejects.toMatchObject({
- response: { code: SEND_PACING_LIMITED, retryAfterSeconds: 3 },
+ response: { code: SEND_PACING_LIMITED, retryAfterSeconds: 1 },
});
+ jest.setSystemTime(NOW.getTime() + 12_000);
+
+ // The warm send read no cold count, so it cannot have missed the cold send's row.
+ await expect(service.assertSendAllowed('s1', '[email protected]')).resolves.toBeDefined();
});
it('hands back a send taken until settled at once when it is settled as never sent', async () => {
diff --git a/src/modules/message/send-pacing.service.ts b/src/modules/message/send-pacing.service.ts
index 13be13d93..dfd72662c 100644
--- a/src/modules/message/send-pacing.service.ts
+++ b/src/modules/message/send-pacing.service.ts
@@ -100,7 +100,8 @@ interface BreakerState {
* persisted count and, for every held admission, the count it read plus the admissions held from it
* on: none of those rows can be in the count it read, so nothing is counted twice. An admission stays held
* for as long as any admission taken after it is held, since those may have read before its row landed
- * and would otherwise miss it once it stops being held.
+ * and would otherwise miss it once it stops being held. The cold cap holds its own chain the same way, over
+ * the cold admissions alone: only a cold admission reads a cold count.
*/
interface Admission {
/** Epoch ms its own term ends: a window from its admission, or from its settle. */
@@ -108,7 +109,10 @@ interface Admission {
/** Epoch ms it stops being held, so only its row counts from then on; refreshed by heldAdmissions. */
heldUntil: number;
sentToday: number;
- /** Set for a cold send; the key is folded the way the cold count folds dialects. */
+ /**
+ * Set for a cold send; the key is folded the way the cold count folds dialects. Cleared by heldAdmissions
+ * once no cold admission's own term is running, so from then on only its row counts against the cold cap.
+ */
cold: { key: string; coldToday: number } | null;
}
@@ -323,6 +327,11 @@ export class SendPacingService {
const hold = this.holds.get(sessionId);
if (!hold) return [];
const now = Date.now();
+ // The cold terms lapse as a whole, once no cold admission's own term is running: a warm admission read no
+ // cold count, so it cannot have missed a cold row, and letting it hold them would keep a cold send that
+ // wrote no row counted for as long as warm sends keep arriving. Cleared rather than skipped, so a cold
+ // admission taken later cannot bring them back.
+ if (!hold.admissions.some(a => a.cold && a.until > now)) for (const a of hold.admissions) a.cold = null;
// Newest first: an admission stays held while the next one is. Each one taken after it may have read
// its count before this one's row landed, and its own term counts this one and each later one once, on
// a count read before any of their rows.
@@ -418,7 +427,11 @@ export class SendPacingService {
// Refused only because of cold sends still held: the batch fits once they lapse, not at the next UTC day.
const retryAfter =
coldChats + groupToday + coldCount <= allowance
- ? secondsUntilLapsed(this.heldAdmissions(sessionId, dayStart).filter(a => a.cold))
+ ? secondsUntilLapsed(
+ this.heldAdmissions(sessionId, dayStart)
+ .filter(a => a.cold)
+ .map(a => a.until),
+ )
: secondsUntilNextUtcDay();
this.refuse('cold_daily_cap', sessionId, retryAfter, {
reason:
@@ -540,7 +553,8 @@ export class SendPacingService {
const sentToday = held.reduce((max, a, i) => Math.max(max, a.sentToday + held.length - i), persisted);
if (sentToday < allowance) return;
- const retryAfter = persisted < allowance ? secondsUntilLapsed(held) : secondsUntilNextUtcDay();
+ const retryAfter =
+ persisted < allowance ? secondsUntilLapsed(held.map(a => a.heldUntil)) : secondsUntilNextUtcDay();
this.refuse('daily_cap', sessionId, retryAfter, {
reason: `Daily send allowance of ${allowance} reached for a session ${ageDays} day(s) old`,
allowance,
@@ -632,7 +646,11 @@ export class SendPacingService {
const retryAfter =
persisted + groupToday < allowance
- ? secondsUntilLapsed(this.heldAdmissions(sessionId, dayStart).filter(a => a.cold))
+ ? secondsUntilLapsed(
+ this.heldAdmissions(sessionId, dayStart)
+ .filter(a => a.cold)
+ .map(a => a.until),
+ )
: secondsUntilNextUtcDay();
this.refuse('cold_daily_cap', sessionId, retryAfter, {
reason: `Daily allowance of ${allowance} new conversation(s) reached for a session ${ageDays} day(s) old`,
@@ -758,13 +776,13 @@ function startOfUtcDay(at: Date): Date {
}
/**
- * Seconds until every one of these held admissions could have lapsed: when the persisted count alone is
- * under the cap, the refusal lifts then, not at the next UTC day. One still unsettled lapses a window
- * after its settle at the earliest, so it is hinted as if it settled now.
+ * Seconds until every one of these held admissions could have lapsed, given the epoch ms each one's hold
+ * ends: when the persisted count alone is under the cap, the refusal lifts then, not at the next UTC day.
+ * One still unsettled lapses a window after its settle at the earliest, so it is hinted as if it settled now.
*/
-function secondsUntilLapsed(held: Admission[]): number {
+function secondsUntilLapsed(holdEnds: number[]): number {
const now = Date.now();
- const last = held.reduce((max, a) => Math.max(max, Math.min(a.heldUntil, now + ADMISSION_WINDOW_MS)), now);
+ const last = holdEnds.reduce((max, end) => Math.max(max, Math.min(end, now + ADMISSION_WINDOW_MS)), now);
return Math.max(1, Math.ceil((last - now) / 1000));
}