Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(dashboard): keep the audit export inside the route throttle budget
Yudhi Armyndharis27eb09adbeb3
diff --git a/dashboard/src/pages/Logs.test.ts b/dashboard/src/pages/Logs.test.tsindex 2141e6b44..a9b01217b 100644--- a/dashboard/src/pages/Logs.test.ts+++ b/dashboard/src/pages/Logs.test.ts@@ -156,7 +156,7 @@ test('an export that stops at the row cap says so, counted in the UI language',renderLogs();await screen.findByText('infra.restart');fireEvent.click(screen.getByRole('button', { name: 'Export CSV' }));- await screen.findByText(/covers only the newest 18,000 entries/, {}, { timeout: 10_000 });+ await screen.findByText(/covers only the newest 10,000 entries/, {}, { timeout: 10_000 });assert.equal(downloads.length, 1, 'the rows fetched up to the cap are still downloaded');} finally {exportTotal = null;diff --git a/dashboard/src/utils/fetchAllPages.test.ts b/dashboard/src/utils/fetchAllPages.test.tsindex 9b262d172..f64729704 100644--- a/dashboard/src/utils/fetchAllPages.test.ts+++ b/dashboard/src/utils/fetchAllPages.test.ts@@ -86,7 +86,7 @@ test('a throttled page is retried at the same offset instead of ending the walk');});-test('a page that stays throttled, or fails any other way, still fails the walk', async () => {+test('a first page that stays throttled, or a page failing any other way, fails the walk', async () => {const { fetchPage } = fakeSource(600, 200);let attempts = 0;await assert.rejects(@@ -99,7 +99,9 @@ test('a page that stays throttled, or fails any other way, still fails the walk'),{ status: 429 },);- assert.equal(attempts, 4, 'three retries after the first attempt');+ // Waits of one and then two seconds outlast the one-second tier; a 429 after that is a tier no+ // wait here outlasts, so a further retry only spends the budget that tier is refusing.+ assert.equal(attempts, 3, 'two retries after the first attempt');attempts = 0;await assert.rejects(@@ -116,18 +118,29 @@ test('a page that stays throttled, or fails any other way, still fails the walk'assert.equal(attempts, 2, 'a server error is not retried');});-test('a default walk stops inside the per-minute throttle and says the result stopped short', async () => {- // The gateway allows 100 requests a minute per route and then refuses that route for the whole- // minute, which no retry outlasts. The Logs page's own reads share that budget.+test('a default walk leaves half the per-minute route budget and says the result stopped short', async () => {+ // The gateway allows 100 requests a minute per route and IP and then refuses that route for the+ // whole minute, which no retry outlasts. The Logs page's own reads share that budget.const { fetchPage, calls } = fakeSource(60_000, 200);+ const { items: rows, truncated } = await fetchAllPages(fetchPage, { retryDelayMs: 0 });+ assert.equal(truncated, true);+ assert.ok(calls.length <= 50, `the walk spent ${calls.length} of the 100 requests the minute allows`);+ assert.equal(rows.length, calls.length * 200);+});++test('a page still throttled past the one-second tier ends the walk with the rows in hand', async () => {+ // The per-minute or per-hour tier refused the route: keep what came in rather than wait and throw.+ const { fetchPage, calls } = fakeSource(6_000, 200);+ let attempts = 0;const { items: rows, truncated } = await fetchAllPages(async (limit, offset) => {- if (calls.length >= 100) throw httpError(429);+ attempts++;+ if (offset >= 1_000) throw httpError(429);return fetchPage(limit, offset);},{ retryDelayMs: 0 },);assert.equal(truncated, true);- assert.ok(calls.length <= 90, `the walk spent ${calls.length} of the 100 requests the minute allows`);- assert.equal(rows.length, calls.length * 200);+ assert.equal(rows.length, 1_000);+ assert.equal(attempts, calls.length + 3, 'the refused page was tried more than three times');});diff --git a/dashboard/src/utils/fetchAllPages.ts b/dashboard/src/utils/fetchAllPages.tsindex de00817e8..87b8a09a2 100644--- a/dashboard/src/utils/fetchAllPages.ts+++ b/dashboard/src/utils/fetchAllPages.ts@@ -10,12 +10,14 @@ interface FetchAllPagesOptions {retryDelayMs?: number;}-// The gateway's per-IP throttler allows 10 requests a second by default, so a walk of more than ten-// pages trips it. A few doubling waits outlast that one-second window. The per-minute tier (100-// requests per route, after which the route is refused for a whole minute) cannot be outlasted, so-// the default `maxItems` stops a walk at 90 pages of 200 and leaves the caller's own reads of that-// route part of the budget.-const THROTTLE_RETRIES = 3;+// The gateway throttles each route per IP in three tiers by default: 10 requests a second, 100 a+// minute and 1000 an hour, and a tripped tier refuses the route for its whole window. A walk of+// more than ten pages trips the one-second tier, which a wait of one second, then two, outlasts. A+// 429 still there after both comes from the minute or hour tier, which no wait here outlasts, so the+// walk stops instead of spending more of the budget that tier is refusing. The default `maxItems`+// stops a walk at 50 pages of 200: half the minute's budget stays for the caller's own reads of that+// route, and an hour's budget holds about 20 such walks.+const THROTTLE_RETRIES = 2;async function retryThrottled<T>(fetchOnce: () => Promise<T>, delayMs: number): Promise<T> {for (let attempt = 0; ; attempt++) {@@ -36,16 +38,24 @@ async function retryThrottled<T>(fetchOnce: () => Promise<T>, delayMs: number):* caps at MAX_AUDIT_PAGE_SIZE), so a `page.length < requested` test reads a clamped first page as* "last page" and silently truncates the result.*- * `truncated` is set when the `maxItems` safety cap ended the walk while the server still had rows.+ * `truncated` is set when the `maxItems` safety cap ended the walk while the server still had rows,+ * or when a page stayed throttled after rows were already in hand.*/export async function fetchAllPages<T>(fetchPage: (limit: number, offset: number) => Promise<Page<T>>,- { pageSize = 200, maxItems = 18_000, retryDelayMs = 1000 }: FetchAllPagesOptions = {},+ { pageSize = 200, maxItems = 10_000, retryDelayMs = 1000 }: FetchAllPagesOptions = {},): Promise<{ items: T[]; truncated: boolean }> {const all: T[] = [];let offset = 0;for (;;) {- const { data, total } = await retryThrottled(() => fetchPage(pageSize, offset), retryDelayMs);+ let page: Page<T>;+ try {+ page = await retryThrottled(() => fetchPage(pageSize, offset), retryDelayMs);+ } catch (err) {+ if (all.length > 0 && (err as { status?: number }).status === 429) return { items: all, truncated: true };+ throw err;+ }+ const { data, total } = page;all.push(...data);offset += data.length;if (data.length === 0 || offset >= total) return { items: all, truncated: false };