Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(dashboard): export the audit CSV in full or say why it stopped

Yudhi Armyndhariscdb1e7c46e93
diff --git a/dashboard/src/i18n/locales/ar.json b/dashboard/src/i18n/locales/ar.json
index b689c1a8e..733300a0a 100644
--- a/dashboard/src/i18n/locales/ar.json
+++ b/dashboard/src/i18n/locales/ar.json
@@ -690,7 +690,9 @@
"filteredTitle": "لا نتائج مطابقة في هذه الصفحة",
"filteredDescription": "يبحث حقل البحث في الصفحة الحالية فقط. امسحه أو تنقّل بين الصفحات أو استخدم مرشح الخطورة للتصفية من الخادم.",
"filteredServerDescription": "لا سجلات تطابق هذه المرشحات. عدّل مرشح الخطورة لتوسيع البحث."
- }
+ },
+ "exportFailed": "تعذّر تصدير سجل التدقيق",
+ "exportTruncated": "يغطي التصدير أحدث {{rows}} إدخال فقط. ضيّق مرشح الخطورة لتصدير الإدخالات الأقدم."
},
"messageTester": {
"title": "اختبار الرسائل",
diff --git a/dashboard/src/i18n/locales/de.json b/dashboard/src/i18n/locales/de.json
index 1915c832f..904612c06 100644
--- a/dashboard/src/i18n/locales/de.json
+++ b/dashboard/src/i18n/locales/de.json
@@ -673,7 +673,9 @@
"filteredTitle": "Keine Treffer auf dieser Seite",
"filteredDescription": "Die Suche filtert nur die aktuelle Seite. Leere sie, blättere weiter oder grenze serverseitig über den Severity-Filter ein.",
"filteredServerDescription": "Keine Protokolle passen auf diese Filter. Passe den Severity-Filter an, um die Suche zu erweitern."
- }
+ },
+ "exportFailed": "Audit-Protokoll konnte nicht exportiert werden",
+ "exportTruncated": "Der Export umfasst nur die neuesten {{rows}} Einträge. Grenze den Severity-Filter ein, um ältere Einträge zu exportieren."
},
"messageTester": {
"title": "Nachrichtentester",
diff --git a/dashboard/src/i18n/locales/en.json b/dashboard/src/i18n/locales/en.json
index 97acf5095..d96270e87 100644
--- a/dashboard/src/i18n/locales/en.json
+++ b/dashboard/src/i18n/locales/en.json
@@ -673,7 +673,9 @@
"filteredTitle": "No matches on this page",
"filteredDescription": "The search filters the current page only. Clear it, page through, or use the severity filter to narrow server-side.",
"filteredServerDescription": "No logs match these filters. Adjust the severity filter to widen the search."
- }
+ },
+ "exportFailed": "Couldn't export the audit log",
+ "exportTruncated": "The export covers only the newest {{rows}} entries. Narrow the severity filter to export older ones."
},
"messageTester": {
"title": "Message Tester",
diff --git a/dashboard/src/i18n/locales/es.json b/dashboard/src/i18n/locales/es.json
index 3208195bc..484d7102a 100644
--- a/dashboard/src/i18n/locales/es.json
+++ b/dashboard/src/i18n/locales/es.json
@@ -628,7 +628,9 @@
"filteredTitle": "Sin coincidencias en esta página",
"filteredDescription": "La búsqueda filtra solo la página actual. Bórrala, navega entre páginas o usa el filtro de severidad para acotar en el servidor.",
"filteredServerDescription": "Ningún registro coincide con estos filtros. Ajusta el filtro de severidad para ampliar la búsqueda."
- }
+ },
+ "exportFailed": "No se pudo exportar el registro de auditoría",
+ "exportTruncated": "La exportación solo abarca las {{rows}} entradas más recientes. Restringe el filtro de severidad para exportar las más antiguas."
},
"messageTester": {
"title": "Probador de mensajes",
diff --git a/dashboard/src/i18n/locales/fr.json b/dashboard/src/i18n/locales/fr.json
index dd94dae29..9092d9771 100644
--- a/dashboard/src/i18n/locales/fr.json
+++ b/dashboard/src/i18n/locales/fr.json
@@ -677,7 +677,9 @@
"filteredTitle": "Aucune correspondance sur cette page",
"filteredDescription": "La recherche ne filtre que la page actuelle. Effacez-la, naviguez entre les pages ou utilisez le filtre de gravité côté serveur.",
"filteredServerDescription": "Aucun journal ne correspond à ces filtres. Ajustez le filtre de gravité pour élargir la recherche."
- }
+ },
+ "exportFailed": "Impossible d'exporter le journal d'audit",
+ "exportTruncated": "L'export ne couvre que les {{rows}} entrées les plus récentes. Affinez le filtre de gravité pour exporter les plus anciennes."
},
"messageTester": {
"title": "Testeur de messages",
diff --git a/dashboard/src/i18n/locales/he.json b/dashboard/src/i18n/locales/he.json
index 04f09028e..2c1c1ed8a 100644
--- a/dashboard/src/i18n/locales/he.json
+++ b/dashboard/src/i18n/locales/he.json
@@ -682,7 +682,9 @@
"filteredTitle": "אין התאמות בעמוד זה",
"filteredDescription": "החיפוש מסנן את העמוד הנוכחי בלבד. נקו אותו, דפדפו בין העמודים או השתמשו במסנן החומרה בצד השרת.",
"filteredServerDescription": "אין יומנים התואמים למסננים אלה. התאימו את מסנן החומרה כדי להרחיב את החיפוש."
- }
+ },
+ "exportFailed": "לא ניתן לייצא את יומן הביקורת",
+ "exportTruncated": "הייצוא כולל רק את {{rows}} הרשומות האחרונות. צמצמו את מסנן החומרה כדי לייצא רשומות ישנות יותר."
},
"messageTester": {
"title": "בדיקת הודעות",
diff --git a/dashboard/src/i18n/locales/hi.json b/dashboard/src/i18n/locales/hi.json
index fa9d8850b..ece82b0e7 100644
--- a/dashboard/src/i18n/locales/hi.json
+++ b/dashboard/src/i18n/locales/hi.json
@@ -673,7 +673,9 @@
"filteredTitle": "इस पृष्ठ पर कोई मेल नहीं",
"filteredDescription": "खोज केवल वर्तमान पृष्ठ को फ़िल्टर करती है। इसे साफ़ करें, अन्य पृष्ठ देखें, या सर्वर पर ही परिणाम सीमित करने के लिए गंभीरता फ़िल्टर का उपयोग करें।",
"filteredServerDescription": "इन फ़िल्टर से कोई लॉग मेल नहीं खाता। खोज को व्यापक बनाने के लिए गंभीरता फ़िल्टर को समायोजित करें।"
- }
+ },
+ "exportFailed": "ऑडिट लॉग निर्यात नहीं किया जा सका",
+ "exportTruncated": "निर्यात में केवल नवीनतम {{rows}} प्रविष्टियाँ शामिल हैं। पुरानी प्रविष्टियाँ निर्यात करने के लिए गंभीरता फ़िल्टर को सीमित करें।"
},
"messageTester": {
"title": "संदेश परीक्षक",
diff --git a/dashboard/src/i18n/locales/it.json b/dashboard/src/i18n/locales/it.json
index f356a642a..51cf7529d 100644
--- a/dashboard/src/i18n/locales/it.json
+++ b/dashboard/src/i18n/locales/it.json
@@ -677,7 +677,9 @@
"filteredTitle": "Nessuna corrispondenza in questa pagina",
"filteredDescription": "La ricerca filtra solo la pagina corrente. Cancellala, sfoglia le pagine o usa il filtro severità lato server.",
"filteredServerDescription": "Nessun log corrisponde a questi filtri. Modifica il filtro severità per ampliare la ricerca."
- }
+ },
+ "exportFailed": "Impossibile esportare il log di controllo",
+ "exportTruncated": "L'esportazione copre solo le {{rows}} voci più recenti. Restringi il filtro severità per esportare quelle più vecchie."
},
"messageTester": {
"title": "Messaggio di prova",
diff --git a/dashboard/src/i18n/locales/ko.json b/dashboard/src/i18n/locales/ko.json
index 2ad3d333a..5fa04dd5a 100644
--- a/dashboard/src/i18n/locales/ko.json
+++ b/dashboard/src/i18n/locales/ko.json
@@ -673,7 +673,9 @@
"filteredTitle": "이 페이지에 일치하는 항목이 없습니다",
"filteredDescription": "검색은 현재 페이지만 필터링합니다. 지우거나 페이지를 이동하거나 서버 측 심각도 필터를 사용하세요.",
"filteredServerDescription": "이 필터에 일치하는 로그가 없습니다. 검색 범위를 넓히려면 심각도 필터를 조정하세요."
- }
+ },
+ "exportFailed": "감사 로그를 내보내지 못했습니다",
+ "exportTruncated": "내보내기에는 최신 항목 {{rows}}개만 포함됩니다. 이전 항목을 내보내려면 심각도 필터를 좁히세요."
},
"messageTester": {
"title": "메시지 테스터",
diff --git a/dashboard/src/i18n/locales/pt-BR.json b/dashboard/src/i18n/locales/pt-BR.json
index c865d56e0..9c901feab 100644
--- a/dashboard/src/i18n/locales/pt-BR.json
+++ b/dashboard/src/i18n/locales/pt-BR.json
@@ -628,7 +628,9 @@
"filteredTitle": "Nenhuma correspondência nesta página",
"filteredDescription": "A busca filtra apenas a página atual. Limpe-a, navegue pelas páginas ou use o filtro de severidade no servidor.",
"filteredServerDescription": "Nenhum log corresponde a estes filtros. Ajuste o filtro de severidade para ampliar a busca."
- }
+ },
+ "exportFailed": "Não foi possível exportar o registro de auditoria",
+ "exportTruncated": "A exportação cobre apenas as {{rows}} entradas mais recentes. Restrinja o filtro de severidade para exportar as mais antigas."
},
"messageTester": {
"title": "Testador de mensagens",
diff --git a/dashboard/src/i18n/locales/te.json b/dashboard/src/i18n/locales/te.json
index 433907d6d..84316774e 100644
--- a/dashboard/src/i18n/locales/te.json
+++ b/dashboard/src/i18n/locales/te.json
@@ -673,7 +673,9 @@
"filteredTitle": "ఈ పేజీలో సరిపోలినవి లేవు",
"filteredDescription": "శోధన ప్రస్తుత పేజీని మాత్రమే ఫిల్టర్ చేస్తుంది. దాన్ని క్లియర్ చేయండి, పేజీల మధ్య మారండి లేదా సర్వర్ వైపు తీవ్రత ఫిల్టర్‌ను ఉపయోగించండి.",
"filteredServerDescription": "ఈ ఫిల్టర్‌లకు సరిపోయే లాగ్‌లు లేవు. వెతుకులను విస్తరించడానికి తీవ్రత ఫిల్టర్‌ను సర్దుబాటు చేయండి."
- }
+ },
+ "exportFailed": "ఆడిట్ లాగ్‌ను ఎక్స్‌పోర్ట్ చేయలేకపోయాం",
+ "exportTruncated": "ఎక్స్‌పోర్ట్ తాజా {{rows}} ఎంట్రీలను మాత్రమే కవర్ చేస్తుంది. పాత ఎంట్రీలను ఎక్స్‌పోర్ట్ చేయడానికి తీవ్రత ఫిల్టర్‌ను కుదించండి."
},
"messageTester": {
"title": "మెసేజ్ టెస్టర్",
diff --git a/dashboard/src/i18n/locales/tr.json b/dashboard/src/i18n/locales/tr.json
index 6af33e80f..9f9277d15 100644
--- a/dashboard/src/i18n/locales/tr.json
+++ b/dashboard/src/i18n/locales/tr.json
@@ -673,7 +673,9 @@
"filteredTitle": "Bu sayfada eşleşme yok",
"filteredDescription": "Arama yalnızca geçerli sayfayı filtreler. Temizleyin, sayfalar arasında gezinin veya sunucu tarafında önem filtresini kullanın.",
"filteredServerDescription": "Bu filtrelerle eşleşen log yok. Aramayı genişletmek için önem filtresini ayarlayın."
- }
+ },
+ "exportFailed": "Denetim kaydı dışa aktarılamadı",
+ "exportTruncated": "Dışa aktarma yalnızca en yeni {{rows}} kaydı kapsar. Daha eski kayıtları dışa aktarmak için önem filtresini daraltın."
},
"messageTester": {
"title": "Mesaj Test Aracı",
diff --git a/dashboard/src/i18n/locales/zh-CN.json b/dashboard/src/i18n/locales/zh-CN.json
index ccde41100..a2819f105 100644
--- a/dashboard/src/i18n/locales/zh-CN.json
+++ b/dashboard/src/i18n/locales/zh-CN.json
@@ -673,7 +673,9 @@
"filteredTitle": "本页无匹配项",
"filteredDescription": "搜索仅过滤当前页。请清除搜索、翻页,或使用严重级别筛选在服务端收窄。",
"filteredServerDescription": "没有符合这些筛选条件的日志。请调整严重级别筛选以扩大搜索范围。"
- }
+ },
+ "exportFailed": "无法导出审计日志",
+ "exportTruncated": "导出仅涵盖最新的 {{rows}} 条记录。请缩小严重级别筛选范围以导出更早的条目。"
},
"messageTester": {
"title": "消息测试器",
diff --git a/dashboard/src/i18n/locales/zh-HK.json b/dashboard/src/i18n/locales/zh-HK.json
index f410df628..647d83113 100644
--- a/dashboard/src/i18n/locales/zh-HK.json
+++ b/dashboard/src/i18n/locales/zh-HK.json
@@ -673,7 +673,9 @@
"filteredTitle": "本頁無符合項目",
"filteredDescription": "搜尋只會過濾目前頁面。請清除搜尋、翻頁,或使用嚴重程度篩選在伺服器端收窄。",
"filteredServerDescription": "沒有符合這些篩選條件的日誌。請調整嚴重程度篩選以擴大搜尋範圍。"
- }
+ },
+ "exportFailed": "無法匯出審計記錄",
+ "exportTruncated": "匯出僅涵蓋最新的 {{rows}} 條記錄。請收窄嚴重程度篩選範圍以匯出較早的項目。"
},
"messageTester": {
"title": "訊息測試器",
diff --git a/dashboard/src/pages/Logs.test.ts b/dashboard/src/pages/Logs.test.ts
index c0498b133..2141e6b44 100644
--- a/dashboard/src/pages/Logs.test.ts
+++ b/dashboard/src/pages/Logs.test.ts
@@ -5,7 +5,7 @@
// renders null fields with its fallbacks rather than crashing, and that the search filter matches
// action and errorMessage case-insensitively over rows whose other fields are null.
import '../test-helpers/register-hooks.ts';
-import { test, before } from 'node:test';
+import { test, before, afterEach } from 'node:test';
import assert from 'node:assert/strict';
import { createElement } from 'react';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
@@ -51,13 +51,46 @@ function jsonResponse(data: unknown): Response {
return new Response(JSON.stringify(data), { headers: { 'Content-Type': 'application/json' } });
}
+// When set, the export's page walk (limit=200) fails with this status; the on-screen page still loads.
+let exportFailure: number | null = null;
+// When set, the export's page walk sees a table of this many rows, served 200 at a time.
+let exportTotal: number | null = null;
+
function installFetchStub(): void {
- globalThis.fetch = ((): Promise<Response> =>
- Promise.resolve(jsonResponse({ data: LOGS, total: LOGS.length }))) as typeof fetch;
+ globalThis.fetch = ((input: RequestInfo | URL): Promise<Response> => {
+ const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url;
+ if (exportFailure && url.includes('limit=200')) {
+ return Promise.resolve(new Response(JSON.stringify({ message: 'boom' }), { status: exportFailure }));
+ }
+ if (exportTotal && url.includes('limit=200')) {
+ return Promise.resolve(jsonResponse({ data: Array(200).fill(LOG_FAILED_SEND), total: exportTotal }));
+ }
+ return Promise.resolve(jsonResponse({ data: LOGS, total: LOGS.length }));
+ }) as typeof fetch;
+}
+
+/** Record the export's blobs instead of handing them to jsdom, which has no object URLs. */
+function recordDownloads(): { downloads: Blob[]; restore: () => void } {
+ const downloads: Blob[] = [];
+ const realCreate = URL.createObjectURL;
+ const realRevoke = URL.revokeObjectURL;
+ URL.createObjectURL = (blob: Blob) => {
+ downloads.push(blob);
+ return 'blob:export';
+ };
+ URL.revokeObjectURL = () => {};
+ return {
+ downloads,
+ restore: () => {
+ URL.createObjectURL = realCreate;
+ URL.revokeObjectURL = realRevoke;
+ },
+ };
}
let rtl: typeof import('@testing-library/react');
let Logs: () => React.ReactElement;
+let ToastProvider: (typeof import('../components/Toast.tsx'))['ToastProvider'];
before(async () => {
const { installJsdomGlobals } = await import('../test-helpers/jsdom.ts');
@@ -66,12 +99,17 @@ before(async () => {
const { i18nReady } = await import('../i18n/index.ts');
await i18nReady;
rtl = await import('@testing-library/react');
+ ({ ToastProvider } = await import('../components/Toast.tsx'));
({ Logs } = await import('./Logs.tsx'));
});
+afterEach(() => rtl.cleanup());
+
function renderLogs(): HTMLElement {
const client = new QueryClient({ defaultOptions: { queries: { retry: false } } });
- return rtl.render(createElement(QueryClientProvider, { client }, createElement(Logs))).container;
+ return rtl.render(
+ createElement(QueryClientProvider, { client }, createElement(ToastProvider, null, createElement(Logs))),
+ ).container;
}
test('the log table renders rows whose nullable fields are null without crashing, and shows both severities', async () => {
@@ -84,3 +122,45 @@ test('the log table renders rows whose nullable fields are null without crashing
assert.ok(container.textContent?.includes('—'), 'null ip falls back to the dash placeholder');
assert.ok(!container.textContent?.includes('null'), 'no raw null leaks into the DOM');
});
+
+test('a failed full export is reported, and the rows on screen are not downloaded in its place', async () => {
+ const { screen, fireEvent } = rtl;
+ const { downloads, restore } = recordDownloads();
+ exportFailure = 500;
+ try {
+ renderLogs();
+ await screen.findByText('infra.restart');
+ fireEvent.click(screen.getByRole('button', { name: 'Export CSV' }));
+ await screen.findByText("Couldn't export the audit log");
+ assert.equal(downloads.length, 0, 'the current page was downloaded as if it were the full export');
+ } finally {
+ exportFailure = null;
+ restore();
+ }
+});
+
+test('an export that stops at the row cap says so, counted in the UI language', async () => {
+ const { screen, fireEvent } = rtl;
+ const { downloads, restore } = recordDownloads();
+ // Stand in for a machine whose default locale groups digits with a dot, as de-DE does.
+ const realToLocaleString = Number.prototype.toLocaleString;
+ Number.prototype.toLocaleString = function (
+ this: number,
+ locales?: Intl.LocalesArgument,
+ options?: Intl.NumberFormatOptions,
+ ) {
+ return realToLocaleString.call(this, locales ?? 'de-DE', options);
+ };
+ exportTotal = 60_000;
+ try {
+ renderLogs();
+ await screen.findByText('infra.restart');
+ fireEvent.click(screen.getByRole('button', { name: 'Export CSV' }));
+ await screen.findByText(/covers only the newest 18,000 entries/, {}, { timeout: 10_000 });
+ assert.equal(downloads.length, 1, 'the rows fetched up to the cap are still downloaded');
+ } finally {
+ exportTotal = null;
+ Number.prototype.toLocaleString = realToLocaleString;
+ restore();
+ }
+});
diff --git a/dashboard/src/pages/Logs.tsx b/dashboard/src/pages/Logs.tsx
index 750ed40a8..1a02c16c4 100644
--- a/dashboard/src/pages/Logs.tsx
+++ b/dashboard/src/pages/Logs.tsx
@@ -4,6 +4,7 @@ import { Download, Search, Filter, Loader2, FileText, AlertCircle } from 'lucide
import type { AuditLog } from '../services/api';
import { auditApi } from '../services/api';
import { useDocumentTitle } from '../hooks/useDocumentTitle';
+import { useToast } from '../hooks/useToast';
import { useLogsQuery } from '../hooks/queries';
import { PageHeader } from '../components/PageHeader';
import { CustomSelect } from '../components/CustomSelect';
@@ -13,7 +14,8 @@ import { escapeCsvCell } from '../utils/csv';
import './Logs.css';
export function Logs() {
- const { t } = useTranslation();
+ const { t, i18n } = useTranslation();
+ const toast = useToast();
useDocumentTitle(t('logs.title'));
const [searchQuery, setSearchQuery] = useState('');
const [severityFilter, setSeverityFilter] = useState('all');
@@ -88,12 +90,12 @@ export function Logs() {
// Export the WHOLE audit history (honouring the active severity filter + search), not just the
// current page — paginate through the API up to a safety cap so a huge table can't OOM the tab. On
- // a fetch error, fall back to exporting the rows already on screen.
+ // a fetch error, report it and download nothing: the rows on screen would pass for the full export.
const handleExportCsv = async () => {
if (exporting) return;
setExporting(true);
try {
- const all = await fetchAllPages<AuditLog>((limit, offset) =>
+ const { items: all, truncated } = await fetchAllPages<AuditLog>((limit, offset) =>
auditApi.list({ severity: severityParam, limit, offset }),
);
const q = searchQuery.toLowerCase();
@@ -101,8 +103,13 @@ export function Logs() {
? all.filter(l => l.action.toLowerCase().includes(q) || (l.errorMessage || '').toLowerCase().includes(q))
: all;
if (rows.length > 0) download(buildCsv(rows));
- } catch {
- if (filteredLogs.length > 0) download(buildCsv(filteredLogs)); // graceful fallback to the page
+ // The cap keeps the newest rows (the API orders newest first); older ones are missing. The count
+ // follows the UI language, not the browser's locale, so it reads right inside the sentence.
+ if (truncated) {
+ toast.warning(t('logs.exportTruncated', { rows: all.length.toLocaleString(i18n.resolvedLanguage) }));
+ }
+ } catch (err) {
+ toast.error(t('logs.exportFailed'), err instanceof Error ? err.message : undefined);
} finally {
setExporting(false);
}
diff --git a/dashboard/src/utils/fetchAllPages.test.ts b/dashboard/src/utils/fetchAllPages.test.ts
index 994e458d7..9b262d172 100644
--- a/dashboard/src/utils/fetchAllPages.test.ts
+++ b/dashboard/src/utils/fetchAllPages.test.ts
@@ -17,15 +17,16 @@ function fakeSource(total: number, serverMax: number) {
test('walks every page when the server clamps below the requested size', async () => {
// The shipped bug: requesting 500 against a server that clamps to 200 stopped after one page.
const { fetchPage, calls } = fakeSource(650, 200);
- const rows = await fetchAllPages(fetchPage, { pageSize: 500 });
+ const { items: rows, truncated } = await fetchAllPages(fetchPage, { pageSize: 500 });
assert.equal(rows.length, 650);
assert.deepEqual(rows.slice(0, 3), [0, 1, 2]);
assert.equal(calls.length, 4);
+ assert.equal(truncated, false);
});
test('stops once total is reached without an extra empty request', async () => {
const { fetchPage, calls } = fakeSource(400, 200);
- const rows = await fetchAllPages(fetchPage, { pageSize: 200 });
+ const { items: rows } = await fetchAllPages(fetchPage, { pageSize: 200 });
assert.equal(rows.length, 400);
assert.equal(calls.length, 2);
});
@@ -33,7 +34,7 @@ test('stops once total is reached without an extra empty request', async () => {
test('stops on an empty page even if total over-reports', async () => {
// Guards the infinite loop: a stale/wrong `total` must not keep the loop spinning.
const { fetchPage, calls } = fakeSource(50, 200);
- const rows = await fetchAllPages(async (limit, offset) => {
+ const { items: rows } = await fetchAllPages(async (limit, offset) => {
const page = await fetchPage(limit, offset);
return { data: page.data, total: 9999 };
});
@@ -41,15 +42,92 @@ test('stops on an empty page even if total over-reports', async () => {
assert.equal(calls.length, 2);
});
-test('honours the safety cap', async () => {
+test('honours the safety cap, and says the result stopped short', async () => {
const { fetchPage } = fakeSource(10_000, 200);
- const rows = await fetchAllPages(fetchPage, { pageSize: 200, maxItems: 500 });
+ const { items: rows, truncated } = await fetchAllPages(fetchPage, { pageSize: 200, maxItems: 500 });
assert.equal(rows.length, 600); // stops at the first page that crosses the cap
+ assert.equal(truncated, true);
+});
+
+test('a walk that ends exactly at the cap is not truncated', async () => {
+ const { fetchPage } = fakeSource(600, 200);
+ const { items: rows, truncated } = await fetchAllPages(fetchPage, { pageSize: 200, maxItems: 600 });
+ assert.equal(rows.length, 600);
+ assert.equal(truncated, false);
});
test('returns an empty list when there is nothing to export', async () => {
const { fetchPage, calls } = fakeSource(0, 200);
- const rows = await fetchAllPages(fetchPage);
+ const { items: rows } = await fetchAllPages(fetchPage);
assert.deepEqual(rows, []);
assert.equal(calls.length, 1);
});
+
+/** An error shaped like the API client's: the HTTP status rides on the Error. */
+function httpError(status: number): Error & { status: number } {
+ return Object.assign(new Error(`HTTP ${status}`), { status });
+}
+
+test('a throttled page is retried at the same offset instead of ending the walk', async () => {
+ // The per-IP throttler allows 10 requests a second, so a large export trips it partway through.
+ const { fetchPage, calls } = fakeSource(600, 200);
+ let throttled = 2;
+ const { items: rows } = await fetchAllPages(
+ async (limit, offset) => {
+ if (offset === 400 && throttled-- > 0) throw httpError(429);
+ return fetchPage(limit, offset);
+ },
+ { retryDelayMs: 0 },
+ );
+ assert.equal(rows.length, 600);
+ assert.deepEqual(
+ calls.map(c => c.offset),
+ [0, 200, 400],
+ );
+});
+
+test('a page that stays throttled, or fails any other way, still fails the walk', async () => {
+ const { fetchPage } = fakeSource(600, 200);
+ let attempts = 0;
+ await assert.rejects(
+ fetchAllPages(
+ async () => {
+ attempts++;
+ throw httpError(429);
+ },
+ { retryDelayMs: 0 },
+ ),
+ { status: 429 },
+ );
+ assert.equal(attempts, 4, 'three retries after the first attempt');
+
+ attempts = 0;
+ await assert.rejects(
+ fetchAllPages(
+ async (limit, offset) => {
+ attempts++;
+ if (offset === 200) throw httpError(500);
+ return fetchPage(limit, offset);
+ },
+ { retryDelayMs: 0 },
+ ),
+ { status: 500 },
+ );
+ assert.equal(attempts, 2, 'a server error is not retried');
+});
+
+test('a default walk stops inside the per-minute throttle and says the result stopped short', async () => {
+ // The gateway allows 100 requests a minute per route and then refuses that route for the whole
+ // minute, which no retry outlasts. The Logs page's own reads share that budget.
+ const { fetchPage, calls } = fakeSource(60_000, 200);
+ const { items: rows, truncated } = await fetchAllPages(
+ async (limit, offset) => {
+ if (calls.length >= 100) throw httpError(429);
+ return fetchPage(limit, offset);
+ },
+ { retryDelayMs: 0 },
+ );
+ assert.equal(truncated, true);
+ assert.ok(calls.length <= 90, `the walk spent ${calls.length} of the 100 requests the minute allows`);
+ assert.equal(rows.length, calls.length * 200);
+});
diff --git a/dashboard/src/utils/fetchAllPages.ts b/dashboard/src/utils/fetchAllPages.ts
index 9aa6f88c2..de00817e8 100644
--- a/dashboard/src/utils/fetchAllPages.ts
+++ b/dashboard/src/utils/fetchAllPages.ts
@@ -6,6 +6,26 @@ interface Page<T> {
interface FetchAllPagesOptions {
pageSize?: number;
maxItems?: number;
+ /** Wait before the first retry of a throttled (429) page; doubles on each further retry. */
+ retryDelayMs?: number;
+}
+
+// The gateway's per-IP throttler allows 10 requests a second by default, so a walk of more than ten
+// pages trips it. A few doubling waits outlast that one-second window. The per-minute tier (100
+// requests per route, after which the route is refused for a whole minute) cannot be outlasted, so
+// the default `maxItems` stops a walk at 90 pages of 200 and leaves the caller's own reads of that
+// route part of the budget.
+const THROTTLE_RETRIES = 3;
+
+async function retryThrottled<T>(fetchOnce: () => Promise<T>, delayMs: number): Promise<T> {
+ for (let attempt = 0; ; attempt++) {
+ try {
+ return await fetchOnce();
+ } catch (err) {
+ if ((err as { status?: number }).status !== 429 || attempt >= THROTTLE_RETRIES) throw err;
+ await new Promise(resolve => setTimeout(resolve, delayMs * 2 ** attempt));
+ }
+ }
}
/**
@@ -15,18 +35,20 @@ interface FetchAllPagesOptions {
* the returned page length against the *requested* size. Endpoints clamp `limit` server-side (audit
* caps at MAX_AUDIT_PAGE_SIZE), so a `page.length < requested` test reads a clamped first page as
* "last page" and silently truncates the result.
+ *
+ * `truncated` is set when the `maxItems` safety cap ended the walk while the server still had rows.
*/
export async function fetchAllPages<T>(
fetchPage: (limit: number, offset: number) => Promise<Page<T>>,
- { pageSize = 200, maxItems = 50_000 }: FetchAllPagesOptions = {},
-): Promise<T[]> {
+ { pageSize = 200, maxItems = 18_000, retryDelayMs = 1000 }: FetchAllPagesOptions = {},
+): Promise<{ items: T[]; truncated: boolean }> {
const all: T[] = [];
let offset = 0;
for (;;) {
- const { data, total } = await fetchPage(pageSize, offset);
+ const { data, total } = await retryThrottled(() => fetchPage(pageSize, offset), retryDelayMs);
all.push(...data);
offset += data.length;
- if (data.length === 0 || offset >= total || all.length >= maxItems) break;
+ if (data.length === 0 || offset >= total) return { items: all, truncated: false };
+ if (all.length >= maxItems) return { items: all, truncated: true };
}
- return all;
}