Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep malformed backup timestamps out of ages and task order
pulse-triage[bot]caedc0574e83
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.mdindex 9bef7ed10b..4be081caf1 100644--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md@@ -7928,6 +7928,17 @@ explains the uncertainty and retains every possible latest task's status, start,duration, literal error and full ID. Possible tasks are displayed in a stablenon-time order; older known tasks are not possible latest tasks.+Backup artifact and task times must be real calendar/time values with an explicit+RFC3339 zone, matching the API's `time.Time` JSON. Do not let the browser normalise+September 31, accept 24:00 or assume a timezone for an unzoned/date-only value.+Malformed times retain their raw observations and unknown age/order; they do not+drop a backup, imply failed completion, supply a duration or hide an undated+failure behind a dated OK. Valid offsets, leap days and fractional seconds remain+usable. `proxmoxBackupTimestamp.test.ts` and `backupTimestampEvidence.test.tsx`+cover all three artifact sources and tasks through the parsed inventory boundary,+including ordinary-poll recovery. Actual browser proof covers the changed model+through Coverage, By date, detail and activity consumers in different timezones.+The task column remains visible even when no latest task can be established.Existing workload/search scope includes candidate errors and IDs, never anotherconnection's namesake. Ordinary inventory refresh can restore an ordered latest,diff --git a/frontend-modern/src/features/proxmox/__tests__/backupTimestampEvidence.test.tsx b/frontend-modern/src/features/proxmox/__tests__/backupTimestampEvidence.test.tsxnew file mode 100644index 0000000000..8dab802cf8--- /dev/null+++ b/frontend-modern/src/features/proxmox/__tests__/backupTimestampEvidence.test.tsx@@ -0,0 +1,163 @@+import { cleanup, fireEvent, render, screen, waitFor, within } from '@solidjs/testing-library';+import { Route, Router } from '@solidjs/router';+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import { resetCreateNonSuspendingQueryCacheForTest } from '@/hooks/createNonSuspendingQuery';+import { protectionPolicy } from '@/utils/__fixtures__/protectionPosture';+import { ProxmoxBackupsTable } from '../ProxmoxBackupsTable';+import {+ BACKUP_IDENTITY_NOW,+ identityArchive,+ identityGuest,+ identityPBS,+ identitySnapshot,+ identityTask,+} from '../__fixtures__/backupIdentity';++const api = vi.hoisted(() => ({ fetch: vi.fn(), json: vi.fn(), width: 390 }));+vi.mock('@/utils/apiClient', () => ({ apiFetch: api.fetch, apiFetchJSON: api.json }));+vi.mock('@/hooks/useObservedElementWidth', () => ({+ useObservedElementWidth: () => ({ setElement: () => {}, width: () => api.width }),+}));+const guest = identityGuest('east-100', 'east', 'pve-1');+let repaired: boolean;+const response = (data: unknown) =>+ new Response(JSON.stringify({ data }), { headers: { 'Content-Type': 'application/json' } });+function mount() {+ api.fetch.mockImplementation((url: string) =>+ Promise.resolve(+ response(+ url === '/api/backups/pbs'+ ? {+ backups: [+ identityPBS({+ backupTime: repaired ? '2026-10-04T10:00:00Z' : '2026-09-31T10:00:00Z',+ }),+ ],+ }+ : {+ backupTasks: [+ identityTask({+ id: 'malformed-failure',+ status: 'failed',+ error: 'Retained failure',+ startTime: repaired ? '2026-10-04T09:00:00Z' : '2026-10-04T09:00:00',+ }),+ identityTask({ id: 'dated-ok' }),+ ],+ storageBackups: [+ identityArchive({ time: repaired ? '2026-10-04T11:00:00+02:00' : '2026-10-04' }),+ ],+ guestSnapshots: [+ identitySnapshot({+ time: repaired ? '2026-10-04T11:30:00.123456789Z' : '2026-09-31T11:30:00Z',+ }),+ ],+ },+ ),+ ),+ );+ api.json.mockResolvedValue({ data: [], policy: protectionPolicy });+ render(() => (+ <Router>+ <Route+ path="/*"+ component={() => <ProxmoxBackupsTable emptyIcon={<span />} workloads={[guest]} />}+ />+ </Router>+ ));+}+const section = (panel: HTMLElement, label: string) =>+ within(panel).getByText(label, { exact: true, selector: 'th' }).closest('tbody')!;+const value = (parent: HTMLElement, label: string) =>+ within(parent).getByText(label, { exact: true, selector: 'td' }).closest('tr')!.lastElementChild!;+const evidenceTable = (panel: HTMLElement) =>+ within(+ within(panel).getByText('Restore evidence', { exact: true }).closest('div')!.parentElement!,+ ).getByRole('table');+async function open() {+ const toggle = await screen.findByRole('button', { name: /Expand details for east-100/ });+ fireEvent.click(toggle);+ return screen.findByTestId('proxmox-coverage-detail');+}+beforeEach(() => {+ vi.resetAllMocks();+ vi.useFakeTimers({ now: BACKUP_IDENTITY_NOW, toFake: ['Date', 'setInterval', 'clearInterval'] });+ resetCreateNonSuspendingQueryCacheForTest();+ api.width = 390;+ repaired = false;+ window.history.replaceState({}, '', '/proxmox/backups/coverage');+});+afterEach(() => {+ cleanup();+ resetCreateNonSuspendingQueryCacheForTest();+ vi.useRealTimers();+});++describe('malformed timestamp evidence through the parsed HTTP consumer', () => {+ it.each([1200, 390, 320])('does not guess coverage ages or latest OK at %ipx', async (width) => {+ api.width = width;+ mount();+ const panel = await open();+ expect(value(section(panel, 'Task order'), 'Latest task')).toHaveTextContent('Unknown');+ expect(value(section(panel, 'Possible latest task 1'), 'Started')).toHaveTextContent('Unknown');+ expect(value(section(panel, 'Possible latest task 1'), 'Duration')).toHaveTextContent(+ 'Not reported',+ );+ expect(value(section(panel, 'Possible latest task 1'), 'Error')).toHaveTextContent(+ 'Retained failure',+ );+ expect(value(section(panel, 'Possible latest task 2'), 'Status')).toHaveTextContent('OK');+ const evidence = evidenceTable(panel);+ expect(within(evidence).getAllByText('Unknown', { exact: true })).toHaveLength(3);+ expect(document.body).toHaveTextContent('3 completed restore points');+ expect(screen.getByRole('button', { name: '1 unknown' })).toBeInTheDocument();+ expect(api.fetch).toHaveBeenCalledTimes(2);+ });++ it.each([1200, 390, 320])(+ 'retains every undated restore point without chart activity at %ipx',+ async (width) => {+ window.history.replaceState({}, '', '/proxmox/backups/date');+ api.width = width;+ mount();+ await waitFor(() =>+ expect(document.querySelectorAll('[data-proxmox-backup-row="recoverable"]')).toHaveLength(+ 3,+ ),+ );+ const rows = Array.from(+ document.querySelectorAll<HTMLElement>('[data-proxmox-backup-row="recoverable"]'),+ );+ for (const row of rows)+ expect(within(row).getByText('Unknown', { exact: true })).toBeInTheDocument();+ expect(document.body).toHaveTextContent('No completed restore points with known dates');+ expect(document.body).toHaveTextContent('Their ages are unknown.');+ fireEvent.click(within(rows[0]).getByRole('button', { name: /Expand details/ }));+ const panel = await screen.findByTestId('proxmox-recoverable-detail');+ expect(value(section(panel, 'Restore point'), 'Created')).toHaveTextContent('Unknown');+ expect(panel).not.toHaveTextContent('Invalid Date');+ expect(api.fetch).toHaveBeenCalledTimes(2);+ },+ );++ it('restores valid dates and ordered tasks on the ordinary poll without replacing the open panel', async () => {+ mount();+ const panel = await open();+ const toggle = screen.getByRole('button', { name: /Collapse details for east-100/ });+ toggle.focus();+ expect(value(section(panel, 'Task order'), 'Latest task')).toHaveTextContent('Unknown');+ repaired = true;+ await vi.advanceTimersByTimeAsync(30_000);+ await waitFor(() =>+ expect(value(section(panel, 'Latest task'), 'Task ID')).toHaveTextContent('dated-ok'),+ );+ expect(within(panel).queryByText('Task order', { exact: true, selector: 'th' })).toBeNull();+ expect(panel).not.toHaveTextContent('Retained failure');+ const evidence = evidenceTable(panel);+ expect(within(evidence).queryByText('Unknown', { exact: true })).toBeNull();+ expect(screen.getByTestId('proxmox-coverage-detail')).toBe(panel);+ expect(document.activeElement).toBe(toggle);+ expect(screen.getByRole('button', { name: '1 unknown' })).toBeInTheDocument();+ expect(api.fetch).toHaveBeenCalledTimes(4);+ });+});diff --git a/frontend-modern/src/features/proxmox/__tests__/proxmoxBackupTimestamp.test.ts b/frontend-modern/src/features/proxmox/__tests__/proxmoxBackupTimestamp.test.tsnew file mode 100644index 0000000000..7c282057d5--- /dev/null+++ b/frontend-modern/src/features/proxmox/__tests__/proxmoxBackupTimestamp.test.ts@@ -0,0 +1,151 @@+import { describe, expect, it } from 'vitest';+import type { BackupTask } from '@/types/api';+import {+ buildProxmoxBackupRecoveryModel,+ coverageRowMatchesSearch,+} from '../proxmoxBackupRecoveryModel';+import { parsePBSBackupInventory, parsePVEBackupInventory } from '../proxmoxBackupInventory';+import {+ BACKUP_IDENTITY_NOW,+ identityArchive,+ identityGuest,+ identityPBS,+ identitySnapshot,+ identityTask,+} from '../__fixtures__/backupIdentity';++const malformed = [+ '2026-09-31T11:00:00Z',+ '2026-02-29T11:00:00Z',+ '1900-02-29T11:00:00Z',+ '2026-04-31T11:00:00+02:00',+ '2026-00-01T11:00:00Z',+ '2026-10-00T11:00:00Z',+ '2026-10-04T24:00:00Z',+ '2026-10-04T11:60:00Z',+ '2026-10-04T11:00:60Z',+ '2026-10-04T11:00:00+24:00',+ '2026-10-04T11:00:00+00:60',+ '2026-10-04T11:00:00',+ '2026-10-04',+ '10/04/2026',+ '04 Oct 2026 11:00:00 GMT',+ '2026-10-04T11:00:00.1234567890Z',+];+const valid = [+ ['2026-10-04T11:00:00Z', '2026-10-04T11:00:00.000Z'],+ ['2026-10-04T11:00:00+02:30', '2026-10-04T08:30:00.000Z'],+ ['2026-10-03T23:00:00-03:30', '2026-10-04T02:30:00.000Z'],+ ['2026-10-04T11:00:00+00:00', '2026-10-04T11:00:00.000Z'],+ ['2026-10-04T11:00:00-00:00', '2026-10-04T11:00:00.000Z'],+ ['2026-10-04T11:00:00.1Z', '2026-10-04T11:00:00.100Z'],+ ['2026-10-04T11:00:00.123456789Z', '2026-10-04T11:00:00.123Z'],+ ['2024-02-29T23:59:59Z', '2024-02-29T23:59:59.000Z'],+ ['2000-02-29T00:00:00Z', '2000-02-29T00:00:00.000Z'],+] as const;+const kinds = ['pbs', 'archive', 'snapshot'] as const;+type Kind = (typeof kinds)[number];+const guest = identityGuest('east-100', 'east', 'pve-1');++function build(kind: Kind, date: string, tasks: BackupTask[] = []) {+ // Real JSON boundary: malformed dates remain observations, not a rejected+ // whole inventory, a dropped row or guessed values from ctime / other dates.+ const pve = parsePVEBackupInventory({+ data: {+ backupTasks: tasks,+ storageBackups: kind === 'archive' ? [identityArchive({ time: date })] : [],+ guestSnapshots: kind === 'snapshot' ? [identitySnapshot({ time: date })] : [],+ },+ });+ const pbs = parsePBSBackupInventory({+ data: {+ backups: kind === 'pbs' ? [identityPBS({ backupTime: date })] : [],+ },+ });+ return buildProxmoxBackupRecoveryModel({+ workloads: [guest],+ pbsBackups: pbs.backups,+ archives: pve.storageBackups,+ snapshots: pve.guestSnapshots,+ tasks: pve.backupTasks,+ nowMs: BACKUP_IDENTITY_NOW,+ });+}++describe.each(kinds)('%s backup timestamp evidence', (kind) => {+ it.each(malformed)('keeps non-RFC3339 or impossible dates unknown: %s', (date) => {+ const model = build(kind, date);+ const [artifact] = model.recoverableArtifacts;+ expect(artifact.createdAt).toBe(date);+ expect(artifact.createdMs).toBeUndefined();+ expect(model.coverageSummary.recoverableArtifacts).toBe(1);+ expect(model.coverageSummary.totalBytes).toBe(artifact.size ?? 0);+ const row = model.coverageRows[0];+ expect(row.artifacts).toHaveLength(1);+ expect(row.latestBackup).toBeUndefined();+ expect(row.ageUnknown?.[kind]).toBe(true);+ expect(row.workload.resourceId).toBe(guest.id);+ expect(row.posture).toBe('unknown');+ });++ it.each(valid)('valid-date control: preserves %s as %s', (date, utc) => {+ const model = build(kind, date);+ expect(model.recoverableArtifacts[0].createdAt).toBe(date);+ expect(model.recoverableArtifacts[0].createdMs).toBe(Date.parse(utc));+ expect(model.coverageRows[0].ageUnknown?.[kind]).toBe(false);+ expect(model.coverageSummary.recoverableArtifacts).toBe(1);+ });+});++describe('task timestamp evidence', () => {+ it.each(malformed.flatMap((date) => [false, true].map((reverse) => ({ date, reverse }))))(+ 'never lets a guessed start hide a failure behind dated OK: $date reversed=$reverse',+ ({ date, reverse }) => {+ const tasks = [+ identityTask({+ id: 'malformed-failure',+ startTime: date,+ status: 'failed',+ error: 'Retain this failure',+ }),+ identityTask({ id: 'dated-ok' }),+ ];+ const row = build('archive', '2026-10-04T11:00:00Z', reverse ? tasks.reverse() : tasks)+ .coverageRows[0];+ expect(row.latestTask).toBeUndefined();+ expect(row.taskCandidates?.map((task) => task.id)).toEqual(['malformed-failure', 'dated-ok']);+ expect(row.taskCandidates?.[0]).toMatchObject({+ startedAt: date,+ label: 'Failed',+ error: 'Retain this failure',+ instance: 'east',+ node: 'pve-1',+ });+ expect(row.taskCandidates?.[0].startedMs).toBeUndefined();+ expect(row.taskCandidates?.[0].durationSeconds).toBeUndefined();+ expect(coverageRowMatchesSearch(row, 'retain this failure')).toBe(true);+ },+ );++ it.each(malformed)('does not invent a duration from a malformed end: %s', (endTime) => {+ const row = build('archive', '2026-10-04T11:00:00Z', [+ identityTask({ startTime: '2026-01-01T00:00:00Z', endTime }),+ ]).coverageRows[0];+ expect(row.latestTask?.durationSeconds).toBeUndefined();+ expect(row.latestTask?.label).toBe('OK');+ });++ it('valid-date control: orders instants rather than local clock text and measures across offsets', () => {+ const row = build('archive', '2026-10-04T11:00:00Z', [+ identityTask({ id: 'earlier', startTime: '2026-10-04T11:00:00+02:00' }),+ identityTask({+ id: 'later',+ startTime: '2026-10-04T10:00:00Z',+ endTime: '2026-10-04T12:05:00+02:00',+ }),+ ]).coverageRows[0];+ expect(row.latestTask?.id).toBe('later');+ expect(row.latestTask?.durationSeconds).toBe(300);+ expect(row.taskCandidates).toBeUndefined();+ });+});diff --git a/frontend-modern/src/features/proxmox/proxmoxBackupRecoveryModel.ts b/frontend-modern/src/features/proxmox/proxmoxBackupRecoveryModel.tsindex 9a1ae1c325..b68f548c14 100644--- a/frontend-modern/src/features/proxmox/proxmoxBackupRecoveryModel.ts+++ b/frontend-modern/src/features/proxmox/proxmoxBackupRecoveryModel.ts@@ -167,6 +167,37 @@ export function isCompletedRecoveryArtifact(artifact: RecoverableArtifact): boolfunction parseTimestampMs(value: string | undefined): number | undefined {if (!value) return undefined;+ // The API's time.Time values carry an explicit RFC3339 zone. Date.parse+ // alone also accepts browser-local dates and normalises impossible dates+ // (September 31 becomes October 1). Neither is evidence for age or order.+ const parts =+ /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d{1,9})?(Z|[+-]\d{2}:\d{2})$/.exec(+ value,+ );+ if (!parts) return undefined;+ const [, yearText, monthText, dayText, hourText, minuteText, secondText, zone] = parts;+ const [year, month, day, hour, minute, second] = [+ yearText,+ monthText,+ dayText,+ hourText,+ minuteText,+ secondText,+ ].map(Number);+ const leapYear = year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0);+ const daysInMonth = [31, leapYear ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];+ if (+ month < 1 ||+ month > 12 ||+ day < 1 ||+ day > daysInMonth[month - 1] ||+ hour > 23 ||+ minute > 59 ||+ second > 59 ||+ (zone !== 'Z' && (Number(zone.slice(1, 3)) > 23 || Number(zone.slice(4)) > 59))+ ) {+ return undefined;+ }const ms = Date.parse(value);return Number.isFinite(ms) ? ms : undefined;}