Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(dashboard): export the audit CSV in full or say why it stopped
Yudhi Armyndhariscdb1e7c46e93
diff --git a/dashboard/src/i18n/locales/ar.json b/dashboard/src/i18n/locales/ar.jsonindex b689c1a8e..733300a0a 100644--- a/dashboard/src/i18n/locales/ar.json+++ b/dashboard/src/i18n/locales/ar.json@@ -690,7 +690,9 @@"filteredTitle": "لا نتائج مطابقة في هذه الصفحة","filteredDescription": "يبحث حقل البحث في الصفحة الحالية فقط. امسحه أو تنقّل بين الصفحات أو استخدم مرشح الخطورة للتصفية من الخادم.","filteredServerDescription": "لا سجلات تطابق هذه المرشحات. عدّل مرشح الخطورة لتوسيع البحث."- }+ },+ "exportFailed": "تعذّر تصدير سجل التدقيق",+ "exportTruncated": "يغطي التصدير أحدث {{rows}} إدخال فقط. ضيّق مرشح الخطورة لتصدير الإدخالات الأقدم."},"messageTester": {"title": "اختبار الرسائل",diff --git a/dashboard/src/i18n/locales/de.json b/dashboard/src/i18n/locales/de.jsonindex 1915c832f..904612c06 100644--- a/dashboard/src/i18n/locales/de.json+++ b/dashboard/src/i18n/locales/de.json@@ -673,7 +673,9 @@"filteredTitle": "Keine Treffer auf dieser Seite","filteredDescription": "Die Suche filtert nur die aktuelle Seite. Leere sie, blättere weiter oder grenze serverseitig über den Severity-Filter ein.","filteredServerDescription": "Keine Protokolle passen auf diese Filter. Passe den Severity-Filter an, um die Suche zu erweitern."- }+ },+ "exportFailed": "Audit-Protokoll konnte nicht exportiert werden",+ "exportTruncated": "Der Export umfasst nur die neuesten {{rows}} Einträge. Grenze den Severity-Filter ein, um ältere Einträge zu exportieren."},"messageTester": {"title": "Nachrichtentester",diff --git a/dashboard/src/i18n/locales/en.json b/dashboard/src/i18n/locales/en.jsonindex 97acf5095..d96270e87 100644--- a/dashboard/src/i18n/locales/en.json+++ b/dashboard/src/i18n/locales/en.json@@ -673,7 +673,9 @@"filteredTitle": "No matches on this page","filteredDescription": "The search filters the current page only. Clear it, page through, or use the severity filter to narrow server-side.","filteredServerDescription": "No logs match these filters. Adjust the severity filter to widen the search."- }+ },+ "exportFailed": "Couldn't export the audit log",+ "exportTruncated": "The export covers only the newest {{rows}} entries. Narrow the severity filter to export older ones."},"messageTester": {"title": "Message Tester",diff --git a/dashboard/src/i18n/locales/es.json b/dashboard/src/i18n/locales/es.jsonindex 3208195bc..484d7102a 100644--- a/dashboard/src/i18n/locales/es.json+++ b/dashboard/src/i18n/locales/es.json@@ -628,7 +628,9 @@"filteredTitle": "Sin coincidencias en esta página","filteredDescription": "La búsqueda filtra solo la página actual. Bórrala, navega entre páginas o usa el filtro de severidad para acotar en el servidor.","filteredServerDescription": "Ningún registro coincide con estos filtros. Ajusta el filtro de severidad para ampliar la búsqueda."- }+ },+ "exportFailed": "No se pudo exportar el registro de auditoría",+ "exportTruncated": "La exportación solo abarca las {{rows}} entradas más recientes. Restringe el filtro de severidad para exportar las más antiguas."},"messageTester": {"title": "Probador de mensajes",diff --git a/dashboard/src/i18n/locales/fr.json b/dashboard/src/i18n/locales/fr.jsonindex dd94dae29..9092d9771 100644--- a/dashboard/src/i18n/locales/fr.json+++ b/dashboard/src/i18n/locales/fr.json@@ -677,7 +677,9 @@"filteredTitle": "Aucune correspondance sur cette page","filteredDescription": "La recherche ne filtre que la page actuelle. Effacez-la, naviguez entre les pages ou utilisez le filtre de gravité côté serveur.","filteredServerDescription": "Aucun journal ne correspond à ces filtres. Ajustez le filtre de gravité pour élargir la recherche."- }+ },+ "exportFailed": "Impossible d'exporter le journal d'audit",+ "exportTruncated": "L'export ne couvre que les {{rows}} entrées les plus récentes. Affinez le filtre de gravité pour exporter les plus anciennes."},"messageTester": {"title": "Testeur de messages",diff --git a/dashboard/src/i18n/locales/he.json b/dashboard/src/i18n/locales/he.jsonindex 04f09028e..2c1c1ed8a 100644--- a/dashboard/src/i18n/locales/he.json+++ b/dashboard/src/i18n/locales/he.json@@ -682,7 +682,9 @@"filteredTitle": "אין התאמות בעמוד זה","filteredDescription": "החיפוש מסנן את העמוד הנוכחי בלבד. נקו אותו, דפדפו בין העמודים או השתמשו במסנן החומרה בצד השרת.","filteredServerDescription": "אין יומנים התואמים למסננים אלה. התאימו את מסנן החומרה כדי להרחיב את החיפוש."- }+ },+ "exportFailed": "לא ניתן לייצא את יומן הביקורת",+ "exportTruncated": "הייצוא כולל רק את {{rows}} הרשומות האחרונות. צמצמו את מסנן החומרה כדי לייצא רשומות ישנות יותר."},"messageTester": {"title": "בדיקת הודעות",diff --git a/dashboard/src/i18n/locales/hi.json b/dashboard/src/i18n/locales/hi.jsonindex fa9d8850b..ece82b0e7 100644--- a/dashboard/src/i18n/locales/hi.json+++ b/dashboard/src/i18n/locales/hi.json@@ -673,7 +673,9 @@"filteredTitle": "इस पृष्ठ पर कोई मेल नहीं","filteredDescription": "खोज केवल वर्तमान पृष्ठ को फ़िल्टर करती है। इसे साफ़ करें, अन्य पृष्ठ देखें, या सर्वर पर ही परिणाम सीमित करने के लिए गंभीरता फ़िल्टर का उपयोग करें।","filteredServerDescription": "इन फ़िल्टर से कोई लॉग मेल नहीं खाता। खोज को व्यापक बनाने के लिए गंभीरता फ़िल्टर को समायोजित करें।"- }+ },+ "exportFailed": "ऑडिट लॉग निर्यात नहीं किया जा सका",+ "exportTruncated": "निर्यात में केवल नवीनतम {{rows}} प्रविष्टियाँ शामिल हैं। पुरानी प्रविष्टियाँ निर्यात करने के लिए गंभीरता फ़िल्टर को सीमित करें।"},"messageTester": {"title": "संदेश परीक्षक",diff --git a/dashboard/src/i18n/locales/it.json b/dashboard/src/i18n/locales/it.jsonindex f356a642a..51cf7529d 100644--- a/dashboard/src/i18n/locales/it.json+++ b/dashboard/src/i18n/locales/it.json@@ -677,7 +677,9 @@"filteredTitle": "Nessuna corrispondenza in questa pagina","filteredDescription": "La ricerca filtra solo la pagina corrente. Cancellala, sfoglia le pagine o usa il filtro severità lato server.","filteredServerDescription": "Nessun log corrisponde a questi filtri. Modifica il filtro severità per ampliare la ricerca."- }+ },+ "exportFailed": "Impossibile esportare il log di controllo",+ "exportTruncated": "L'esportazione copre solo le {{rows}} voci più recenti. Restringi il filtro severità per esportare quelle più vecchie."},"messageTester": {"title": "Messaggio di prova",diff --git a/dashboard/src/i18n/locales/ko.json b/dashboard/src/i18n/locales/ko.jsonindex 2ad3d333a..5fa04dd5a 100644--- a/dashboard/src/i18n/locales/ko.json+++ b/dashboard/src/i18n/locales/ko.json@@ -673,7 +673,9 @@"filteredTitle": "이 페이지에 일치하는 항목이 없습니다","filteredDescription": "검색은 현재 페이지만 필터링합니다. 지우거나 페이지를 이동하거나 서버 측 심각도 필터를 사용하세요.","filteredServerDescription": "이 필터에 일치하는 로그가 없습니다. 검색 범위를 넓히려면 심각도 필터를 조정하세요."- }+ },+ "exportFailed": "감사 로그를 내보내지 못했습니다",+ "exportTruncated": "내보내기에는 최신 항목 {{rows}}개만 포함됩니다. 이전 항목을 내보내려면 심각도 필터를 좁히세요."},"messageTester": {"title": "메시지 테스터",diff --git a/dashboard/src/i18n/locales/pt-BR.json b/dashboard/src/i18n/locales/pt-BR.jsonindex c865d56e0..9c901feab 100644--- a/dashboard/src/i18n/locales/pt-BR.json+++ b/dashboard/src/i18n/locales/pt-BR.json@@ -628,7 +628,9 @@"filteredTitle": "Nenhuma correspondência nesta página","filteredDescription": "A busca filtra apenas a página atual. Limpe-a, navegue pelas páginas ou use o filtro de severidade no servidor.","filteredServerDescription": "Nenhum log corresponde a estes filtros. Ajuste o filtro de severidade para ampliar a busca."- }+ },+ "exportFailed": "Não foi possível exportar o registro de auditoria",+ "exportTruncated": "A exportação cobre apenas as {{rows}} entradas mais recentes. Restrinja o filtro de severidade para exportar as mais antigas."},"messageTester": {"title": "Testador de mensagens",diff --git a/dashboard/src/i18n/locales/te.json b/dashboard/src/i18n/locales/te.jsonindex 433907d6d..84316774e 100644--- a/dashboard/src/i18n/locales/te.json+++ b/dashboard/src/i18n/locales/te.json@@ -673,7 +673,9 @@"filteredTitle": "ఈ పేజీలో సరిపోలినవి లేవు","filteredDescription": "శోధన ప్రస్తుత పేజీని మాత్రమే ఫిల్టర్ చేస్తుంది. దాన్ని క్లియర్ చేయండి, పేజీల మధ్య మారండి లేదా సర్వర్ వైపు తీవ్రత ఫిల్టర్ను ఉపయోగించండి.","filteredServerDescription": "ఈ ఫిల్టర్లకు సరిపోయే లాగ్లు లేవు. వెతుకులను విస్తరించడానికి తీవ్రత ఫిల్టర్ను సర్దుబాటు చేయండి."- }+ },+ "exportFailed": "ఆడిట్ లాగ్ను ఎక్స్పోర్ట్ చేయలేకపోయాం",+ "exportTruncated": "ఎక్స్పోర్ట్ తాజా {{rows}} ఎంట్రీలను మాత్రమే కవర్ చేస్తుంది. పాత ఎంట్రీలను ఎక్స్పోర్ట్ చేయడానికి తీవ్రత ఫిల్టర్ను కుదించండి."},"messageTester": {"title": "మెసేజ్ టెస్టర్",diff --git a/dashboard/src/i18n/locales/tr.json b/dashboard/src/i18n/locales/tr.jsonindex 6af33e80f..9f9277d15 100644--- a/dashboard/src/i18n/locales/tr.json+++ b/dashboard/src/i18n/locales/tr.json@@ -673,7 +673,9 @@"filteredTitle": "Bu sayfada eşleşme yok","filteredDescription": "Arama yalnızca geçerli sayfayı filtreler. Temizleyin, sayfalar arasında gezinin veya sunucu tarafında önem filtresini kullanın.","filteredServerDescription": "Bu filtrelerle eşleşen log yok. Aramayı genişletmek için önem filtresini ayarlayın."- }+ },+ "exportFailed": "Denetim kaydı dışa aktarılamadı",+ "exportTruncated": "Dışa aktarma yalnızca en yeni {{rows}} kaydı kapsar. Daha eski kayıtları dışa aktarmak için önem filtresini daraltın."},"messageTester": {"title": "Mesaj Test Aracı",diff --git a/dashboard/src/i18n/locales/zh-CN.json b/dashboard/src/i18n/locales/zh-CN.jsonindex ccde41100..a2819f105 100644--- a/dashboard/src/i18n/locales/zh-CN.json+++ b/dashboard/src/i18n/locales/zh-CN.json@@ -673,7 +673,9 @@"filteredTitle": "本页无匹配项","filteredDescription": "搜索仅过滤当前页。请清除搜索、翻页,或使用严重级别筛选在服务端收窄。","filteredServerDescription": "没有符合这些筛选条件的日志。请调整严重级别筛选以扩大搜索范围。"- }+ },+ "exportFailed": "无法导出审计日志",+ "exportTruncated": "导出仅涵盖最新的 {{rows}} 条记录。请缩小严重级别筛选范围以导出更早的条目。"},"messageTester": {"title": "消息测试器",diff --git a/dashboard/src/i18n/locales/zh-HK.json b/dashboard/src/i18n/locales/zh-HK.jsonindex f410df628..647d83113 100644--- a/dashboard/src/i18n/locales/zh-HK.json+++ b/dashboard/src/i18n/locales/zh-HK.json@@ -673,7 +673,9 @@"filteredTitle": "本頁無符合項目","filteredDescription": "搜尋只會過濾目前頁面。請清除搜尋、翻頁,或使用嚴重程度篩選在伺服器端收窄。","filteredServerDescription": "沒有符合這些篩選條件的日誌。請調整嚴重程度篩選以擴大搜尋範圍。"- }+ },+ "exportFailed": "無法匯出審計記錄",+ "exportTruncated": "匯出僅涵蓋最新的 {{rows}} 條記錄。請收窄嚴重程度篩選範圍以匯出較早的項目。"},"messageTester": {"title": "訊息測試器",diff --git a/dashboard/src/pages/Logs.test.ts b/dashboard/src/pages/Logs.test.tsindex c0498b133..2141e6b44 100644--- a/dashboard/src/pages/Logs.test.ts+++ b/dashboard/src/pages/Logs.test.ts@@ -5,7 +5,7 @@// renders null fields with its fallbacks rather than crashing, and that the search filter matches// action and errorMessage case-insensitively over rows whose other fields are null.import '../test-helpers/register-hooks.ts';-import { test, before } from 'node:test';+import { test, before, afterEach } from 'node:test';import assert from 'node:assert/strict';import { createElement } from 'react';import { QueryClient, QueryClientProvider } from '@tanstack/react-query';@@ -51,13 +51,46 @@ function jsonResponse(data: unknown): Response {return new Response(JSON.stringify(data), { headers: { 'Content-Type': 'application/json' } });}+// When set, the export's page walk (limit=200) fails with this status; the on-screen page still loads.+let exportFailure: number | null = null;+// When set, the export's page walk sees a table of this many rows, served 200 at a time.+let exportTotal: number | null = null;+function installFetchStub(): void {- globalThis.fetch = ((): Promise<Response> =>- Promise.resolve(jsonResponse({ data: LOGS, total: LOGS.length }))) as typeof fetch;+ globalThis.fetch = ((input: RequestInfo | URL): Promise<Response> => {+ const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;+ if (exportFailure && url.includes('limit=200')) {+ return Promise.resolve(new Response(JSON.stringify({ message: 'boom' }), { status: exportFailure }));+ }+ if (exportTotal && url.includes('limit=200')) {+ return Promise.resolve(jsonResponse({ data: Array(200).fill(LOG_FAILED_SEND), total: exportTotal }));+ }+ return Promise.resolve(jsonResponse({ data: LOGS, total: LOGS.length }));+ }) as typeof fetch;+}++/** Record the export's blobs instead of handing them to jsdom, which has no object URLs. */+function recordDownloads(): { downloads: Blob[]; restore: () => void } {+ const downloads: Blob[] = [];+ const realCreate = URL.createObjectURL;+ const realRevoke = URL.revokeObjectURL;+ URL.createObjectURL = (blob: Blob) => {+ downloads.push(blob);+ return 'blob:export';+ };+ URL.revokeObjectURL = () => {};+ return {+ downloads,+ restore: () => {+ URL.createObjectURL = realCreate;+ URL.revokeObjectURL = realRevoke;+ },+ };}let rtl: typeof import('@testing-library/react');let Logs: () => React.ReactElement;+let ToastProvider: (typeof import('../components/Toast.tsx'))['ToastProvider'];before(async () => {const { installJsdomGlobals } = await import('../test-helpers/jsdom.ts');@@ -66,12 +99,17 @@ before(async () => {const { i18nReady } = await import('../i18n/index.ts');await i18nReady;rtl = await import('@testing-library/react');+ ({ ToastProvider } = await import('../components/Toast.tsx'));({ Logs } = await import('./Logs.tsx'));});+afterEach(() => rtl.cleanup());+function renderLogs(): HTMLElement {const client = new QueryClient({ defaultOptions: { queries: { retry: false } } });- return rtl.render(createElement(QueryClientProvider, { client }, createElement(Logs))).container;+ return rtl.render(+ createElement(QueryClientProvider, { client }, createElement(ToastProvider, null, createElement(Logs))),+ ).container;}test('the log table renders rows whose nullable fields are null without crashing, and shows both severities', async () => {@@ -84,3 +122,45 @@ test('the log table renders rows whose nullable fields are null without crashingassert.ok(container.textContent?.includes('—'), 'null ip falls back to the dash placeholder');assert.ok(!container.textContent?.includes('null'), 'no raw null leaks into the DOM');});++test('a failed full export is reported, and the rows on screen are not downloaded in its place', async () => {+ const { screen, fireEvent } = rtl;+ const { downloads, restore } = recordDownloads();+ exportFailure = 500;+ try {+ renderLogs();+ await screen.findByText('infra.restart');+ fireEvent.click(screen.getByRole('button', { name: 'Export CSV' }));+ await screen.findByText("Couldn't export the audit log");+ assert.equal(downloads.length, 0, 'the current page was downloaded as if it were the full export');+ } finally {+ exportFailure = null;+ restore();+ }+});++test('an export that stops at the row cap says so, counted in the UI language', async () => {+ const { screen, fireEvent } = rtl;+ const { downloads, restore } = recordDownloads();+ // Stand in for a machine whose default locale groups digits with a dot, as de-DE does.+ const realToLocaleString = Number.prototype.toLocaleString;+ Number.prototype.toLocaleString = function (+ this: number,+ locales?: Intl.LocalesArgument,+ options?: Intl.NumberFormatOptions,+ ) {+ return realToLocaleString.call(this, locales ?? 'de-DE', options);+ };+ exportTotal = 60_000;+ try {+ renderLogs();+ await screen.findByText('infra.restart');+ fireEvent.click(screen.getByRole('button', { name: 'Export CSV' }));+ await screen.findByText(/covers only the newest 18,000 entries/, {}, { timeout: 10_000 });+ assert.equal(downloads.length, 1, 'the rows fetched up to the cap are still downloaded');+ } finally {+ exportTotal = null;+ Number.prototype.toLocaleString = realToLocaleString;+ restore();+ }+});diff --git a/dashboard/src/pages/Logs.tsx b/dashboard/src/pages/Logs.tsxindex 750ed40a8..1a02c16c4 100644--- a/dashboard/src/pages/Logs.tsx+++ b/dashboard/src/pages/Logs.tsx@@ -4,6 +4,7 @@ import { Download, Search, Filter, Loader2, FileText, AlertCircle } from 'lucideimport type { AuditLog } from '../services/api';import { auditApi } from '../services/api';import { useDocumentTitle } from '../hooks/useDocumentTitle';+import { useToast } from '../hooks/useToast';import { useLogsQuery } from '../hooks/queries';import { PageHeader } from '../components/PageHeader';import { CustomSelect } from '../components/CustomSelect';@@ -13,7 +14,8 @@ import { escapeCsvCell } from '../utils/csv';import './Logs.css';export function Logs() {- const { t } = useTranslation();+ const { t, i18n } = useTranslation();+ const toast = useToast();useDocumentTitle(t('logs.title'));const [searchQuery, setSearchQuery] = useState('');const [severityFilter, setSeverityFilter] = useState('all');@@ -88,12 +90,12 @@ export function Logs() {// Export the WHOLE audit history (honouring the active severity filter + search), not just the// current page — paginate through the API up to a safety cap so a huge table can't OOM the tab. On- // a fetch error, fall back to exporting the rows already on screen.+ // a fetch error, report it and download nothing: the rows on screen would pass for the full export.const handleExportCsv = async () => {if (exporting) return;setExporting(true);try {- const all = await fetchAllPages<AuditLog>((limit, offset) =>+ const { items: all, truncated } = await fetchAllPages<AuditLog>((limit, offset) =>auditApi.list({ severity: severityParam, limit, offset }),);const q = searchQuery.toLowerCase();@@ -101,8 +103,13 @@ export function Logs() {? all.filter(l => l.action.toLowerCase().includes(q) || (l.errorMessage || '').toLowerCase().includes(q)): all;if (rows.length > 0) download(buildCsv(rows));- } catch {- if (filteredLogs.length > 0) download(buildCsv(filteredLogs)); // graceful fallback to the page+ // The cap keeps the newest rows (the API orders newest first); older ones are missing. The count+ // follows the UI language, not the browser's locale, so it reads right inside the sentence.+ if (truncated) {+ toast.warning(t('logs.exportTruncated', { rows: all.length.toLocaleString(i18n.resolvedLanguage) }));+ }+ } catch (err) {+ toast.error(t('logs.exportFailed'), err instanceof Error ? err.message : undefined);} finally {setExporting(false);}diff --git a/dashboard/src/utils/fetchAllPages.test.ts b/dashboard/src/utils/fetchAllPages.test.tsindex 994e458d7..9b262d172 100644--- a/dashboard/src/utils/fetchAllPages.test.ts+++ b/dashboard/src/utils/fetchAllPages.test.ts@@ -17,15 +17,16 @@ function fakeSource(total: number, serverMax: number) {test('walks every page when the server clamps below the requested size', async () => {// The shipped bug: requesting 500 against a server that clamps to 200 stopped after one page.const { fetchPage, calls } = fakeSource(650, 200);- const rows = await fetchAllPages(fetchPage, { pageSize: 500 });+ const { items: rows, truncated } = await fetchAllPages(fetchPage, { pageSize: 500 });assert.equal(rows.length, 650);assert.deepEqual(rows.slice(0, 3), [0, 1, 2]);assert.equal(calls.length, 4);+ assert.equal(truncated, false);});test('stops once total is reached without an extra empty request', async () => {const { fetchPage, calls } = fakeSource(400, 200);- const rows = await fetchAllPages(fetchPage, { pageSize: 200 });+ const { items: rows } = await fetchAllPages(fetchPage, { pageSize: 200 });assert.equal(rows.length, 400);assert.equal(calls.length, 2);});@@ -33,7 +34,7 @@ test('stops once total is reached without an extra empty request', async () => {test('stops on an empty page even if total over-reports', async () => {// Guards the infinite loop: a stale/wrong `total` must not keep the loop spinning.const { fetchPage, calls } = fakeSource(50, 200);- const rows = await fetchAllPages(async (limit, offset) => {+ const { items: rows } = await fetchAllPages(async (limit, offset) => {const page = await fetchPage(limit, offset);return { data: page.data, total: 9999 };});@@ -41,15 +42,92 @@ test('stops on an empty page even if total over-reports', async () => {assert.equal(calls.length, 2);});-test('honours the safety cap', async () => {+test('honours the safety cap, and says the result stopped short', async () => {const { fetchPage } = fakeSource(10_000, 200);- const rows = await fetchAllPages(fetchPage, { pageSize: 200, maxItems: 500 });+ const { items: rows, truncated } = await fetchAllPages(fetchPage, { pageSize: 200, maxItems: 500 });assert.equal(rows.length, 600); // stops at the first page that crosses the cap+ assert.equal(truncated, true);+});++test('a walk that ends exactly at the cap is not truncated', async () => {+ const { fetchPage } = fakeSource(600, 200);+ const { items: rows, truncated } = await fetchAllPages(fetchPage, { pageSize: 200, maxItems: 600 });+ assert.equal(rows.length, 600);+ assert.equal(truncated, false);});test('returns an empty list when there is nothing to export', async () => {const { fetchPage, calls } = fakeSource(0, 200);- const rows = await fetchAllPages(fetchPage);+ const { items: rows } = await fetchAllPages(fetchPage);assert.deepEqual(rows, []);assert.equal(calls.length, 1);});++/** An error shaped like the API client's: the HTTP status rides on the Error. */+function httpError(status: number): Error & { status: number } {+ return Object.assign(new Error(`HTTP ${status}`), { status });+}++test('a throttled page is retried at the same offset instead of ending the walk', async () => {+ // The per-IP throttler allows 10 requests a second, so a large export trips it partway through.+ const { fetchPage, calls } = fakeSource(600, 200);+ let throttled = 2;+ const { items: rows } = await fetchAllPages(+ async (limit, offset) => {+ if (offset === 400 && throttled-- > 0) throw httpError(429);+ return fetchPage(limit, offset);+ },+ { retryDelayMs: 0 },+ );+ assert.equal(rows.length, 600);+ assert.deepEqual(+ calls.map(c => c.offset),+ [0, 200, 400],+ );+});++test('a page that stays throttled, or fails any other way, still fails the walk', async () => {+ const { fetchPage } = fakeSource(600, 200);+ let attempts = 0;+ await assert.rejects(+ fetchAllPages(+ async () => {+ attempts++;+ throw httpError(429);+ },+ { retryDelayMs: 0 },+ ),+ { status: 429 },+ );+ assert.equal(attempts, 4, 'three retries after the first attempt');++ attempts = 0;+ await assert.rejects(+ fetchAllPages(+ async (limit, offset) => {+ attempts++;+ if (offset === 200) throw httpError(500);+ return fetchPage(limit, offset);+ },+ { retryDelayMs: 0 },+ ),+ { status: 500 },+ );+ assert.equal(attempts, 2, 'a server error is not retried');+});++test('a default walk stops inside the per-minute throttle and says the result stopped short', async () => {+ // The gateway allows 100 requests a minute per route and then refuses that route for the whole+ // minute, which no retry outlasts. The Logs page's own reads share that budget.+ const { fetchPage, calls } = fakeSource(60_000, 200);+ const { items: rows, truncated } = await fetchAllPages(+ async (limit, offset) => {+ if (calls.length >= 100) throw httpError(429);+ return fetchPage(limit, offset);+ },+ { retryDelayMs: 0 },+ );+ assert.equal(truncated, true);+ assert.ok(calls.length <= 90, `the walk spent ${calls.length} of the 100 requests the minute allows`);+ assert.equal(rows.length, calls.length * 200);+});diff --git a/dashboard/src/utils/fetchAllPages.ts b/dashboard/src/utils/fetchAllPages.tsindex 9aa6f88c2..de00817e8 100644--- a/dashboard/src/utils/fetchAllPages.ts+++ b/dashboard/src/utils/fetchAllPages.ts@@ -6,6 +6,26 @@ interface Page<T> {interface FetchAllPagesOptions {pageSize?: number;maxItems?: number;+ /** Wait before the first retry of a throttled (429) page; doubles on each further retry. */+ retryDelayMs?: number;+}++// The gateway's per-IP throttler allows 10 requests a second by default, so a walk of more than ten+// pages trips it. A few doubling waits outlast that one-second window. The per-minute tier (100+// requests per route, after which the route is refused for a whole minute) cannot be outlasted, so+// the default `maxItems` stops a walk at 90 pages of 200 and leaves the caller's own reads of that+// route part of the budget.+const THROTTLE_RETRIES = 3;++async function retryThrottled<T>(fetchOnce: () => Promise<T>, delayMs: number): Promise<T> {+ for (let attempt = 0; ; attempt++) {+ try {+ return await fetchOnce();+ } catch (err) {+ if ((err as { status?: number }).status !== 429 || attempt >= THROTTLE_RETRIES) throw err;+ await new Promise(resolve => setTimeout(resolve, delayMs * 2 ** attempt));+ }+ }}/**@@ -15,18 +35,20 @@ interface FetchAllPagesOptions {* the returned page length against the *requested* size. Endpoints clamp `limit` server-side (audit* caps at MAX_AUDIT_PAGE_SIZE), so a `page.length < requested` test reads a clamped first page as* "last page" and silently truncates the result.+ *+ * `truncated` is set when the `maxItems` safety cap ended the walk while the server still had rows.*/export async function fetchAllPages<T>(fetchPage: (limit: number, offset: number) => Promise<Page<T>>,- { pageSize = 200, maxItems = 50_000 }: FetchAllPagesOptions = {},-): Promise<T[]> {+ { pageSize = 200, maxItems = 18_000, retryDelayMs = 1000 }: FetchAllPagesOptions = {},+): Promise<{ items: T[]; truncated: boolean }> {const all: T[] = [];let offset = 0;for (;;) {- const { data, total } = await fetchPage(pageSize, offset);+ const { data, total } = await retryThrottled(() => fetchPage(pageSize, offset), retryDelayMs);all.push(...data);offset += data.length;- if (data.length === 0 || offset >= total || all.length >= maxItems) break;+ if (data.length === 0 || offset >= total) return { items: all, truncated: false };+ if (all.length >= maxItems) return { items: all, truncated: true };}- return all;}