Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Keep backup-server memory readings faithful to their evidence
pulse-triage[bot]194ef8944e99
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.mdindex 4c824d6011..ed397455bc 100644--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md@@ -21,6 +21,29 @@## Purpose+### Backup-server memory does not invent empty or current readings++The Backup servers table keeps a valid selected memory percentage when a partial+metric reports capacity without used bytes; it never substitutes zero used bytes.+A displayed byte pair needs finite non-negative used bytes and positive capacity,+with used not exceeding capacity. Valid reported pairs still determine the+percentage, including an explicit measured zero. An independent finite percentage+in the range 0–100 remains usable when the byte pair is absent or invalid.++Server reachability does not renew memory evidence. The table reuses the resource+drawer's memory-observation qualification: retained and unknown readings say+`Last known` or `Freshness unknown`; explicitly unavailable readings show no+number. A stopped linked agent qualifies its own reading without invalidating an+independent current source. Offline suppression, datastore risk, backup counts,+identity, History targets and original observation clocks remain unchanged.++Normal-suite `ProxmoxBackupServersTable.memory.test.tsx` covers partial and invalid+metrics, measured zero, retained/unavailable/future observations, stopped-agent+source isolation, live recovery and multi-datastore/offline controls. Browser+proof must consume the resource producer through the actual Backups surface and+its existing drawer at desktop and narrow widths. This improves truthful+inspection, not installed memory relief, a successful backup or guest recovery.+### Fleet protection reads bound their request fan-outCoverage reads keep the existing 200-subject batch size and issue at most fourdiff --git a/frontend-modern/src/features/proxmox/ProxmoxBackupServersTable.tsx b/frontend-modern/src/features/proxmox/ProxmoxBackupServersTable.tsxindex 665fbca130..fc32042be2 100644--- a/frontend-modern/src/features/proxmox/ProxmoxBackupServersTable.tsx+++ b/frontend-modern/src/features/proxmox/ProxmoxBackupServersTable.tsx@@ -26,6 +26,10 @@ import {import type { PBSBackup } from '@/types/api';import type { Resource, ResourcePBSDatastore } from '@/types/resource';import { getPlatformAgentRecord } from '@/utils/agentResources';+import {+ getMemoryObservationPresentation,+ type MemoryObservationPresentation,+} from '@/utils/memoryObservation';import type { StatusIndicatorVariant } from '@/utils/status';import { useObservedElementWidth } from '@/hooks/useObservedElementWidth';@@ -59,6 +63,7 @@ interface BackupServerRow {memoryPercent?: number;memoryUsed?: number;memoryTotal?: number;+ memoryReading?: MemoryObservationPresentation | null;uptimeSeconds?: number;datastore?: ResourcePBSDatastore;backupCount: number;@@ -370,21 +375,45 @@ export function buildBackupServerRows(const datastores = (server.pbs?.datastores ?? []).slice().sort((left, right) => left.name.localeCompare(right.name));- const memoryTotal = server.memory?.total ?? 0;+ const memory = server.memory;+ const memoryTotal =+ typeof memory?.total === 'number' && Number.isFinite(memory.total) && memory.total > 0+ ? memory.total+ : undefined;+ const memoryUsed =+ typeof memory?.used === 'number' &&+ Number.isFinite(memory.used) &&+ memory.used >= 0 &&+ (memoryTotal === undefined || memory.used <= memoryTotal)+ ? memory.used+ : undefined;+ // A partial metric may report a percentage and capacity without used+ // bytes. Capacity cannot turn the missing amount into a measured zero.+ // Keep an independently valid percentage, but show a byte pair only when+ // both amounts were actually reported. Explicit zero remains a reading.+ const percent =+ memoryTotal !== undefined && memoryUsed !== undefined+ ? (memoryUsed / memoryTotal) * 100+ : memory?.current;+ const memoryPercent =+ typeof percent === 'number' && Number.isFinite(percent) && percent >= 0 && percent <= 100+ ? percent+ : undefined;const host = {serverName: server.name,online: serverIsOnline(server),connectionLabel: connectionLabel(server),version: server.pbs?.version,cpuPercent: typeof server.cpu?.current === 'number' ? server.cpu.current : undefined,- memoryPercent:- memoryTotal > 0- ? ((server.memory?.used ?? 0) / memoryTotal) * 100- : typeof server.memory?.current === 'number'- ? server.memory.current- : undefined,- memoryUsed: server.memory?.used,- memoryTotal: memoryTotal > 0 ? memoryTotal : undefined,+ memoryPercent,+ memoryUsed,+ memoryTotal,+ // A healthy PBS connection does not renew a linked host's retained+ // memory. Use the same source-owned observation as the resource drawer.+ memoryReading: getMemoryObservationPresentation(memoryPercent, memory?.observation, false, {+ includeCurrent: false,+ agentStale: (server.agent ?? getPlatformAgentRecord(server))?.stale === true,+ }),uptimeSeconds: server.uptime ?? server.pbs?.uptimeSeconds,};// Each keyed row has a reconciled Solid store. Give it its own JSON DTO@@ -606,22 +635,54 @@ export function ProxmoxBackupServersTable(props: {<TableCell class={getPlatformTableCellClassForKind('numeric-value')}><Show when={!row.continuesServer}><Show- when={row.online && row.memoryPercent !== undefined}- fallback={<span class="text-muted">—</span>}+ when={+ row.online &&+ row.memoryPercent !== undefined &&+ row.memoryReading?.state !== 'unavailable'+ }+ fallback={+ <span class="text-muted text-[11px]">+ {row.online && row.memoryReading?.state === 'unavailable'+ ? 'Unavailable'+ : '—'}+ </span>+ }><spanclass="text-base-content"title={- row.memoryTotal- ? `${formatPlatformTableBytesValue(row.memoryUsed, '0 B')} / ${formatPlatformTableBytesValue(row.memoryTotal)}`- : undefined+ row.memoryReading?.message ??+ (row.memoryTotal !== undefined && row.memoryUsed !== undefined+ ? `${formatPlatformTableBytesValue(row.memoryUsed)} / ${formatPlatformTableBytesValue(row.memoryTotal)}`+ : undefined)}><PlatformTablePercentValue value={row.memoryPercent} /></span>- <Show when={row.memoryTotal && layoutMode() === 'full'}>+ <Show+ when={+ row.memoryTotal !== undefined &&+ row.memoryUsed !== undefined &&+ layoutMode() === 'full'+ }+ ><span class="ml-1 text-[10px] text-muted tabular-nums">- {`(${formatPlatformTableBytesValue(row.memoryUsed, '0 B')}/${formatPlatformTableBytesValue(row.memoryTotal)})`}+ {`(${formatPlatformTableBytesValue(row.memoryUsed)}/${formatPlatformTableBytesValue(row.memoryTotal)})`}+ </span>+ </Show>+ <Show+ when={+ row.memoryReading?.state === 'last-known' ||+ row.memoryReading?.state === 'unknown'+ }+ >+ <span+ class="block whitespace-normal text-[10px] leading-snug text-amber-600 dark:text-amber-300"+ title={row.memoryReading?.message}+ >+ {row.memoryReading?.state === 'last-known'+ ? 'Last known'+ : 'Freshness unknown'}</span></Show></Show>diff --git a/frontend-modern/src/features/proxmox/__tests__/ProxmoxBackupServersTable.memory.test.tsx b/frontend-modern/src/features/proxmox/__tests__/ProxmoxBackupServersTable.memory.test.tsxnew file mode 100644index 0000000000..330e3017b9--- /dev/null+++ b/frontend-modern/src/features/proxmox/__tests__/ProxmoxBackupServersTable.memory.test.tsx@@ -0,0 +1,230 @@+import { cleanup, render, waitFor, within } from '@solidjs/testing-library';+import { createSignal } from 'solid-js';+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';+import type { Resource, ResourceMetric } from '@/types/resource';+import { canonicalizeRealtimeResource } from '@/utils/resourceStateAdapters';+import { buildBackupServerRows, ProxmoxBackupServersTable } from '../ProxmoxBackupServersTable';++vi.mock('@/components/Infrastructure/ResourceDetailDrawer', () => ({+ ResourceDetailDrawer: () => <div>Server details</div>,+}));++const NOW = '2026-10-29T20:00:00Z';+const OBSERVED = '2026-10-29T19:59:00Z';+const server = (memory?: ResourceMetric, extra: Partial<Resource> = {}): Resource =>+ canonicalizeRealtimeResource({+ id: 'pbs-memory',+ type: 'pbs',+ name: 'pbs-memory',+ displayName: 'pbs-memory',+ status: 'online',+ platformId: 'pbs-memory',+ platformType: 'proxmox-pbs',+ sourceType: 'api',+ lastSeen: Date.parse(NOW),+ memory,+ pbs: {+ instanceId: 'pbs-memory',+ connectionHealth: 'healthy',+ datastores: [{ name: 'tank', total: 1000, used: 950, status: 'available' }],+ },+ ...extra,+ });++function memoryCell(container: HTMLElement): HTMLElement {+ const table = container.querySelector('table')!;+ const index = [...table.querySelectorAll('th')].findIndex(+ (head) => head.textContent === 'Memory',+ );+ expect(index).toBeGreaterThan(-1);+ return table.querySelector('tbody tr')!.querySelectorAll('td')[index] as HTMLElement;+}++const show = (resource: Resource, width = 1200) =>+ render(() => <ProxmoxBackupServersTable servers={[resource]} layoutWidth={() => width} />);++beforeEach(() => {+ vi.useFakeTimers({ toFake: ['Date'] });+ vi.setSystemTime(new Date(NOW));+});+afterEach(() => {+ cleanup();+ vi.useRealTimers();+});++describe('PBS memory evidence', () => {+ it('[partial-memory] keeps a percentage-only reading instead of inventing zero used bytes', () => {+ const { container } = show(server({ current: 75, total: 8192 }));+ const cell = memoryCell(container);+ expect(cell.textContent).toBe('75.0%');+ expect(cell.innerHTML).not.toContain('0 B');+ expect(cell.querySelector('[title]')).toBeNull();+ const row = buildBackupServerRows([server({ current: 75, total: 8192 })])[0];+ expect(row.memoryUsed).toBeUndefined();+ expect(row.memoryTotal).toBe(8192);+ expect(row.memoryPercent).toBe(75);+ });++ it('uses reported byte pairs, including measured zero, ahead of inconsistent percentage metadata', () => {+ const positive = show(server({ current: 0, total: 1000, used: 400 }));+ expect(memoryCell(positive.container).textContent).toContain('40.0%');+ expect(memoryCell(positive.container).textContent).toContain('400 B/1000 B');+ positive.unmount();+ const zero = show(server({ current: 99, total: 1000, used: 0 }));+ expect(memoryCell(zero.container).textContent).toBe('0.0%(0 B/1000 B)');+ });++ it('preserves a percentage-only measured zero without claiming a byte amount', () => {+ const { container } = show(server({ current: 0, total: 1000 }));+ expect(memoryCell(container).textContent).toBe('0.0%');+ });++ it.each([undefined, null, '40', -1, 101, NaN, Infinity])(+ 'does not turn invalid or absent percentage %s into a measured zero',+ (current) => {+ const { container } = show(server({ current, total: 1000 } as ResourceMetric));+ expect(memoryCell(container).textContent).toBe('—');+ },+ );++ it.each([+ { used: -1 },+ { used: 1001 },+ { used: NaN },+ { used: Infinity },+ { used: null },+ { used: '400' },+ { total: 0 },+ { total: -1 },+ { total: Infinity },+ { total: '1000' },+ ])('keeps a valid selected percentage without displaying an invalid byte pair %j', (patch) => {+ const { container } = show(+ server({ current: 40, total: 1000, used: 400, ...patch } as ResourceMetric),+ );+ expect(memoryCell(container).textContent).toBe('40.0%');+ });++ it.each([+ ['last-known', OBSERVED, 'Last known'],+ ['new-state', OBSERVED, 'Freshness unknown'],+ ['current', undefined, 'Freshness unknown'],+ ['current', '2026-10-30T20:00:00Z', 'Freshness unknown'],+ ])(+ '[retained-memory] qualifies %s observations without renewing their clock',+ (state, observedAt, label) => {+ const { container } = show(+ server({+ current: 40,+ total: 1000,+ used: 400,+ observation: { state, source: 'agent', observedAt },+ }),+ );+ const cell = memoryCell(container);+ expect(cell.textContent).toContain('40.0%');+ expect(within(cell).getByText(label)).toBeInTheDocument();+ expect(cell.querySelector('[title]')?.getAttribute('title')).toContain(+ 'Not a current measurement',+ );+ expect(within(container).getByText('Healthy')).toBeInTheDocument();+ expect(container.textContent).toContain('95.0%');+ },+ );++ it('[unavailable-memory] hides an explicitly unavailable reading even when old bytes remain', () => {+ const { container } = show(+ server({+ current: 40,+ total: 1000,+ used: 400,+ observation: { state: 'unavailable', source: 'agent', observedAt: OBSERVED },+ }),+ );+ expect(memoryCell(container).textContent).toBe('Unavailable');+ expect(memoryCell(container).innerHTML).not.toContain('400 B');+ expect(container.textContent).toContain('95.0%');+ });++ it('honours a stopped agent without invalidating an independent current memory source', () => {+ const resource = server(+ {+ current: 40,+ observation: { state: 'current', source: 'agent', observedAt: OBSERVED },+ },+ { agent: { agentId: 'host-agent', stale: true } },+ );+ const agent = show(resource);+ expect(within(memoryCell(agent.container)).getByText('Last known')).toBeInTheDocument();+ expect(memoryCell(agent.container).querySelector('[title]')?.getAttribute('title')).toContain(+ 'stopped reporting',+ );+ agent.unmount();+ resource.memory!.observation!.source = 'available-field';+ const independent = show(resource);+ expect(memoryCell(independent.container).textContent).toBe('40.0%');+ });++ it('keeps partial, retained, unavailable and recovered reads on the same live row', async () => {+ const [resources, setResources] = createSignal([server({ current: 75, total: 1000 })]);+ const { container } = render(() => (+ <ProxmoxBackupServersTable servers={resources()} layoutWidth={() => 1200} />+ ));+ const cell = memoryCell(container);+ const row = cell.closest('tr');+ expect(cell.textContent).toBe('75.0%');+ setResources([+ server({+ current: 40,+ total: 1000,+ used: 400,+ observation: {+ state: 'last-known',+ source: 'agent',+ observedAt: OBSERVED,+ },+ }),+ ]);+ await waitFor(() => expect(cell.textContent).toContain('Last known'));+ setResources([+ server({+ current: 40,+ total: 1000,+ used: 400,+ observation: {+ state: 'unavailable',+ source: 'agent',+ observedAt: OBSERVED,+ },+ }),+ ]);+ await waitFor(() => expect(cell.textContent).toBe('Unavailable'));+ setResources([+ server({+ current: 20,+ total: 1000,+ used: 200,+ observation: {+ state: 'current',+ source: 'agent',+ observedAt: OBSERVED,+ },+ }),+ ]);+ await waitFor(() => expect(cell.textContent).toBe('20.0%(200 B/1000 B)'));+ expect(memoryCell(container)).toBe(cell);+ expect(cell.closest('tr')).toBe(row);+ });++ it('keeps host readings suppressed offline and present only once across datastores', () => {+ const resource = server({ current: 40, total: 1000, used: 400 });+ resource.pbs!.datastores!.push({ name: 'offsite', total: 1000, used: 500 });+ const online = show(resource);+ expect(online.container.textContent?.match(/40.0%/g)).toHaveLength(1);+ online.unmount();+ resource.pbs!.connectionHealth = 'unreachable';+ const offline = show(resource);+ expect(memoryCell(offline.container).textContent).toBe('—');+ expect(offline.container.textContent).not.toContain('40.0%');+ });+});