Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(pacing): stop warm sends from holding a cold send's count
Yudhi Armyndharis005412ce5fda
diff --git a/src/modules/message/send-pacing-cold-query.spec.ts b/src/modules/message/send-pacing-cold-query.spec.tsindex d79a90807..56bd45f8d 100644--- a/src/modules/message/send-pacing-cold-query.spec.ts+++ b/src/modules/message/send-pacing-cold-query.spec.ts@@ -667,6 +667,79 @@ describe('concurrent sends against a real database', () => {});});+ describe('a cold send with no row in a busy session', () => {+ const seedKnown = (): Promise<unknown> =>+ ds.query(+ `INSERT INTO "messages" ("id","sessionId","chatId","from","to","type","direction","createdAt")+ VALUES ('seed','s1','[email protected]','a','b','text','incoming',?)`,+ [new Date(NOW.getTime() - 86_400_000).toISOString()],+ );++ // A cold bulk item with an unknown outcome (it never writes a row), a cold send whose row lands,+ // then a warm send every 3 s for 30 s.+ const busySession = async (service: SendPacingService): Promise<void> => {+ await seedKnown();+ const settle = await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });+ settle?.(true);+ expect(await send(service, '[email protected]', 1)).toBe(true);+ for (let t = 1; t <= 10; t++) {+ jest.setSystemTime(NOW.getTime() + t * 3_000);+ expect(await send(service, '[email protected]', 10 + t)).toBe(true);+ }+ };++ it('stops counting it once its own window ends, however busy the session', async () => {+ const service = build([10_000], [2]);+ await busySession(service);++ // One new chat (y) of two reached.+ expect(await send(service, '[email protected]', 99)).toBe(true);+ });++ it('stops charging it to a group add once its own window ends, however busy the session', async () => {+ const service = build([10_000], [2]);+ await busySession(service);++ await expect(service.assertReachoutAllowed('s1', ['[email protected]'])).resolves.toMatchObject({ coldCount: 1 });+ });++ it('judges a second send to that chat against the cap once the first stopped counting', async () => {+ const service = build([10_000], [2]);+ await busySession(service);+ expect(await send(service, '[email protected]', 99)).toBe(true);++ // y and z reached; x would be a third new chat.+ await expect(service.assertSendAllowed('s1', '[email protected]')).rejects.toMatchObject({ status: 429 });+ });+ });++ it.each([+ ['send', (service: SendPacingService) => service.assertSendAllowed('s1', '[email protected]')],+ ['group add', (service: SendPacingService) => service.assertReachoutAllowed('s1', ['[email protected]'])],+ ])('answers a cold %s refused by held sends with a hint a retry then passes', async (_, attempt) => {+ const service = build([10_000], [1]);+ await ds.query(+ `INSERT INTO "messages" ("id","sessionId","chatId","from","to","type","direction","createdAt")+ VALUES ('seed','s1','[email protected]','a','b','text','incoming',?)`,+ [new Date(NOW.getTime() - 86_400_000).toISOString()],+ );+ // A bulk item to a known chat, in flight until +5 s, and a cold send admitted meanwhile whose row never lands.+ const settleKnown = await service.assertSendAllowed('s1', '[email protected]', { untilSettled: true });+ jest.setSystemTime(NOW.getTime() + 1_000);+ await service.assertSendAllowed('s1', '[email protected]');+ jest.setSystemTime(NOW.getTime() + 5_000);+ settleKnown?.(true);+ await insertRow('m1', '[email protected]');+ jest.setSystemTime(NOW.getTime() + 6_000);++ const refusal = (await attempt(service).catch((error: HttpException) => error.getResponse())) as {+ retryAfterSeconds: number;+ };+ jest.setSystemTime(NOW.getTime() + 6_000 + refusal.retryAfterSeconds * 1_000);++ await expect(attempt(service)).resolves.toBeDefined();+ });+it('keeps holding a send taken until settled while it is still running, then for a window from the settle', async () => {const service = build([2], []);expect(await send(service, '[email protected]', 1)).toBe(true);@@ -816,7 +889,7 @@ describe('concurrent sends against a real database', () => {await expect(service.assertSendAllowed('s1', '[email protected]')).rejects.toMatchObject({ status: 429 });});- it('hints a settled cold send as held until the send admitted while it was in flight lapses', async () => {+ it('stops counting a settled cold send once its own window ends, though a warm send admitted meanwhile is held', async () => {const service = build([10_000], [1]);await ds.query(`INSERT INTO "messages" ("id","sessionId","chatId","from","to","type","direction","createdAt")@@ -828,11 +901,14 @@ describe('concurrent sends against a real database', () => {settleCold?.(true);jest.setSystemTime(NOW.getTime() + 5_000);settleKnown?.(true);- jest.setSystemTime(NOW.getTime() + 12_000);-+ jest.setSystemTime(NOW.getTime() + 9_000);await expect(service.assertSendAllowed('s1', '[email protected]')).rejects.toMatchObject({- response: { code: SEND_PACING_LIMITED, retryAfterSeconds: 3 },+ response: { code: SEND_PACING_LIMITED, retryAfterSeconds: 1 },});+ jest.setSystemTime(NOW.getTime() + 12_000);++ // The warm send read no cold count, so it cannot have missed the cold send's row.+ await expect(service.assertSendAllowed('s1', '[email protected]')).resolves.toBeDefined();});it('hands back a send taken until settled at once when it is settled as never sent', async () => {diff --git a/src/modules/message/send-pacing.service.ts b/src/modules/message/send-pacing.service.tsindex 13be13d93..dfd72662c 100644--- a/src/modules/message/send-pacing.service.ts+++ b/src/modules/message/send-pacing.service.ts@@ -100,7 +100,8 @@ interface BreakerState {* persisted count and, for every held admission, the count it read plus the admissions held from it* on: none of those rows can be in the count it read, so nothing is counted twice. An admission stays held* for as long as any admission taken after it is held, since those may have read before its row landed- * and would otherwise miss it once it stops being held.+ * and would otherwise miss it once it stops being held. The cold cap holds its own chain the same way, over+ * the cold admissions alone: only a cold admission reads a cold count.*/interface Admission {/** Epoch ms its own term ends: a window from its admission, or from its settle. */@@ -108,7 +109,10 @@ interface Admission {/** Epoch ms it stops being held, so only its row counts from then on; refreshed by heldAdmissions. */heldUntil: number;sentToday: number;- /** Set for a cold send; the key is folded the way the cold count folds dialects. */+ /**+ * Set for a cold send; the key is folded the way the cold count folds dialects. Cleared by heldAdmissions+ * once no cold admission's own term is running, so from then on only its row counts against the cold cap.+ */cold: { key: string; coldToday: number } | null;}@@ -323,6 +327,11 @@ export class SendPacingService {const hold = this.holds.get(sessionId);if (!hold) return [];const now = Date.now();+ // The cold terms lapse as a whole, once no cold admission's own term is running: a warm admission read no+ // cold count, so it cannot have missed a cold row, and letting it hold them would keep a cold send that+ // wrote no row counted for as long as warm sends keep arriving. Cleared rather than skipped, so a cold+ // admission taken later cannot bring them back.+ if (!hold.admissions.some(a => a.cold && a.until > now)) for (const a of hold.admissions) a.cold = null;// Newest first: an admission stays held while the next one is. Each one taken after it may have read// its count before this one's row landed, and its own term counts this one and each later one once, on// a count read before any of their rows.@@ -418,7 +427,11 @@ export class SendPacingService {// Refused only because of cold sends still held: the batch fits once they lapse, not at the next UTC day.const retryAfter =coldChats + groupToday + coldCount <= allowance- ? secondsUntilLapsed(this.heldAdmissions(sessionId, dayStart).filter(a => a.cold))+ ? secondsUntilLapsed(+ this.heldAdmissions(sessionId, dayStart)+ .filter(a => a.cold)+ .map(a => a.until),+ ): secondsUntilNextUtcDay();this.refuse('cold_daily_cap', sessionId, retryAfter, {reason:@@ -540,7 +553,8 @@ export class SendPacingService {const sentToday = held.reduce((max, a, i) => Math.max(max, a.sentToday + held.length - i), persisted);if (sentToday < allowance) return;- const retryAfter = persisted < allowance ? secondsUntilLapsed(held) : secondsUntilNextUtcDay();+ const retryAfter =+ persisted < allowance ? secondsUntilLapsed(held.map(a => a.heldUntil)) : secondsUntilNextUtcDay();this.refuse('daily_cap', sessionId, retryAfter, {reason: `Daily send allowance of ${allowance} reached for a session ${ageDays} day(s) old`,allowance,@@ -632,7 +646,11 @@ export class SendPacingService {const retryAfter =persisted + groupToday < allowance- ? secondsUntilLapsed(this.heldAdmissions(sessionId, dayStart).filter(a => a.cold))+ ? secondsUntilLapsed(+ this.heldAdmissions(sessionId, dayStart)+ .filter(a => a.cold)+ .map(a => a.until),+ ): secondsUntilNextUtcDay();this.refuse('cold_daily_cap', sessionId, retryAfter, {reason: `Daily allowance of ${allowance} new conversation(s) reached for a session ${ageDays} day(s) old`,@@ -758,13 +776,13 @@ function startOfUtcDay(at: Date): Date {}/**- * Seconds until every one of these held admissions could have lapsed: when the persisted count alone is- * under the cap, the refusal lifts then, not at the next UTC day. One still unsettled lapses a window- * after its settle at the earliest, so it is hinted as if it settled now.+ * Seconds until every one of these held admissions could have lapsed, given the epoch ms each one's hold+ * ends: when the persisted count alone is under the cap, the refusal lifts then, not at the next UTC day.+ * One still unsettled lapses a window after its settle at the earliest, so it is hinted as if it settled now.*/-function secondsUntilLapsed(held: Admission[]): number {+function secondsUntilLapsed(holdEnds: number[]): number {const now = Date.now();- const last = held.reduce((max, a) => Math.max(max, Math.min(a.heldUntil, now + ADMISSION_WINDOW_MS)), now);+ const last = holdEnds.reduce((max, end) => Math.max(max, Math.min(end, now + ADMISSION_WINDOW_MS)), now);return Math.max(1, Math.ceil((last - now) / 1000));}