Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
Bound fleet protection assessment request fan-out
pulse-triage[bot]afc29c6eff39
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.mdindex 6ab8ac41d0..44ea88b769 100644--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md@@ -21,6 +21,27 @@## Purpose+### Fleet protection reads bound their request fan-out++Coverage reads keep the existing 200-subject batch size and issue at most four+posture requests at once per consuming surface. Queued batches start only as a+slot completes, in request order; responses retain that order regardless of+completion order. The entire validated fleet result and policy publish together,+never progressively green partial health. A failed batch aborts active siblings,+discards queued work and retains the initiating failure. A retired read cannot+start queued batches even when a transport resolves after cancellation. Scope,+organisation, replacement-read and disposal ownership remain unchanged, as do+existing polling, auth, policy and independent restore-evidence inspection.++Normal-suite `useProtectionPostures.concurrency.test.ts` checks peak request+counts for a 10,001-subject fleet, out-of-order completion, atomic publication,+failure/retry and cancellation with queued work. Existing hook/cancellation and+Backups consumer suites remain required. Desktop/narrow browser proof must+exercise the actual HTTP producer through Coverage for multi-wave pending,+failure, retry and scope retirement, retaining independent evidence and focus.+This bounds client request pressure, not measured installed CPU relief or+original backup, guest or restore recovery.+### Backup inventory identities must be unambiguousEach PVE task, archive and guest-snapshot collection, and the PBS snapshotdiff --git a/frontend-modern/src/hooks/__tests__/useProtectionPostures.concurrency.test.ts b/frontend-modern/src/hooks/__tests__/useProtectionPostures.concurrency.test.tsnew file mode 100644index 0000000000..be2b16f30b--- /dev/null+++ b/frontend-modern/src/hooks/__tests__/useProtectionPostures.concurrency.test.ts@@ -0,0 +1,194 @@+import { afterEach, describe, expect, it, vi } from 'vitest';+import { createRoot, createSignal } from 'solid-js';+import { resetCreateNonSuspendingQueryCacheForTest } from '@/hooks/createNonSuspendingQuery';+import {+ MAX_CONCURRENT_PROTECTION_POSTURE_REQUESTS,+ useProtectionPostures,+} from '@/hooks/useProtectionPostures';+import { eventBus } from '@/stores/events';+import type { ProtectionPosturesResponse } from '@/types/recovery';+import { protectionPolicy, protectionPosture } from '@/utils/__fixtures__/protectionPosture';++const apiFetchJSONMock = vi.hoisted(() => vi.fn());+vi.mock('@/utils/apiClient', () => ({ apiFetchJSON: apiFetchJSONMock }));++const fleet = (count: number) =>+ Array.from({ length: count }, (_, index) => `vm:${String(index).padStart(5, '0')}`);+const disposers: (() => void)[] = [];+interface PendingBatch {+ ids: string[];+ signal: AbortSignal;+ settled: boolean;+ finish: () => void;+ reject: (reason: unknown) => void;+}++function holdBatches(abortRejects = false) {+ const requests: PendingBatch[] = [];+ let active = 0;+ let peak = 0;+ apiFetchJSONMock.mockImplementation(+ (url: string, { signal }: { signal: AbortSignal }) =>+ new Promise<ProtectionPosturesResponse>((resolve, reject) => {+ const ids = new URL(url, 'https://pulse.invalid').searchParams.getAll('resourceId');+ active += 1;+ peak = Math.max(peak, active);+ const settle = () => {+ if (batch.settled) return false;+ batch.settled = true;+ active -= 1;+ return true;+ };+ const batch: PendingBatch = {+ ids,+ signal,+ settled: false,+ finish: () => {+ if (settle())+ resolve({ data: ids.map(protectionPosture).reverse(), policy: protectionPolicy });+ },+ reject: (error) => {+ if (settle()) reject(error);+ },+ };+ requests.push(batch);+ if (abortRejects)+ signal.addEventListener(+ 'abort',+ () => batch.reject(new DOMException('Cancelled sibling', 'AbortError')),+ { once: true },+ );+ }),+ );+ return { requests, peak: () => peak, active: () => active };+}++function mount(ids: () => readonly string[]) {+ return createRoot((dispose) => {+ disposers.push(dispose);+ return useProtectionPostures(ids);+ });+}++async function flush() {+ // Drain a completed wave without advancing the 30-second polling clock.+ for (let turn = 0; turn < 8; turn++) await Promise.resolve();+}++async function finishRead(requests: PendingBatch[], start = 0) {+ let next = start;+ while (next < requests.length) {+ requests[next++].finish();+ await flush();+ }+}++afterEach(() => {+ disposers.splice(0).forEach((dispose) => dispose());+ apiFetchJSONMock.mockReset();+ resetCreateNonSuspendingQueryCacheForTest();+});++describe('bounded protection posture fan-out', () => {+ it('caps a 10,001-subject read at four active requests and publishes only the complete ordered result', async () => {+ const ids = fleet(10_001);+ const held = holdBatches();+ const hook = mount(() => ids);+ await vi.waitFor(() => expect(held.requests).toHaveLength(4));+ expect(MAX_CONCURRENT_PROTECTION_POSTURE_REQUESTS).toBe(4);+ expect(held.active()).toBe(4);+ expect(hook.postures()).toEqual([]);+ expect(hook.policy().freshnessWindowSeconds).toBe(0);++ // Complete the fourth batch first: exactly one queued batch replaces it.+ held.requests[3].finish();+ await flush();+ expect(held.requests).toHaveLength(5);+ expect(held.active()).toBe(4);+ expect(hook.postures()).toEqual([]);+ expect(hook.resolvedOnce()).toBe(false);+ await finishRead(held.requests);+ await vi.waitFor(() => expect(hook.postures()).toHaveLength(10_001));+ expect(held.peak()).toBe(4);+ expect(held.requests).toHaveLength(51);+ expect(held.requests.flatMap((batch) => batch.ids)).toEqual(ids);+ expect(held.requests.every((batch) => batch.signal === held.requests[0].signal)).toBe(true);+ expect(held.requests[0].signal.aborted).toBe(false);+ expect(hook.postures().map((posture) => posture.subjectResourceId)).toEqual(+ held.requests.flatMap((batch) => batch.ids.slice().reverse()),+ );+ expect(hook.policy()).toEqual(protectionPolicy);+ expect(hook.response.error).toBeNull();+ });++ it('drops queued work and preserves the initiating failure instead of a sibling abort, then retries the whole fleet', async () => {+ const ids = fleet(1_001);+ const held = holdBatches(true);+ const hook = mount(() => ids);+ await vi.waitFor(() => expect(held.requests).toHaveLength(4));+ const failure = new Error('Second batch unavailable');+ held.requests[1].reject(failure);+ await vi.waitFor(() => expect(hook.response.error).toBe(failure));+ expect(held.requests).toHaveLength(4);+ expect(held.requests.every((batch) => batch.signal.aborted)).toBe(true);+ expect(held.active()).toBe(0);+ expect(hook.postures()).toEqual([]);+ expect(hook.policy().freshnessWindowSeconds).toBe(0);++ const retry = hook.refetch();+ expect(held.requests).toHaveLength(8);+ expect(hook.response.error).toBe(failure);+ held.requests[7].finish();+ await flush();+ expect(held.requests).toHaveLength(9);+ expect(hook.postures()).toEqual([]);+ await finishRead(held.requests, 4);+ await retry;+ expect(held.requests.slice(4).flatMap((batch) => batch.ids)).toEqual(ids);+ expect(held.requests).toHaveLength(10);+ expect(held.peak()).toBe(4);+ expect(hook.postures()).toHaveLength(1_001);+ expect(hook.policy()).toEqual(protectionPolicy);+ expect(hook.response.error).toBeNull();+ });++ it.each(['scope', 'organisation', 'replacement', 'empty', 'disposal'] as const)(+ 'never dispatches queued work after %s retirement, even if cancelled transports resolve late',+ async (retirement) => {+ const held = holdBatches();+ const [ids, setIDs] = createSignal<readonly string[]>(fleet(1_001));+ const hook = mount(ids);+ await vi.waitFor(() => expect(held.requests).toHaveLength(4));+ const retired = held.requests.slice();+ let retry: Promise<ProtectionPosturesResponse> | undefined;+ if (retirement === 'scope') setIDs(['vm:replacement']);+ if (retirement === 'organisation') eventBus.emit('org_switched', 'fixture-other-org');+ if (retirement === 'replacement') retry = hook.refetch();+ if (retirement === 'empty') setIDs([]);+ if (retirement === 'disposal') disposers.pop()!();+ await flush();+ expect(retired.every((batch) => batch.signal.aborted)).toBe(true);+ const count = held.requests.length;+ expect(count).toBe(+ retirement === 'scope' ? 5 : retirement === 'empty' || retirement === 'disposal' ? 4 : 8,+ );+ for (const batch of retired) batch.finish();+ await flush();+ expect(held.requests).toHaveLength(count);+ expect(hook.postures()).toEqual([]);+ await finishRead(held.requests, 4);+ await retry;+ if (retirement === 'scope') {+ expect(hook.postures().map((posture) => posture.subjectResourceId)).toEqual([+ 'vm:replacement',+ ]);+ } else if (retirement === 'organisation' || retirement === 'replacement') {+ expect(hook.postures()).toHaveLength(1_001);+ expect(held.requests.slice(4).flatMap((batch) => batch.ids)).toEqual(fleet(1_001));+ } else {+ expect(hook.postures()).toEqual([]);+ }+ expect(hook.response.error).toBeNull();+ },+ );+});diff --git a/frontend-modern/src/hooks/useProtectionPostures.ts b/frontend-modern/src/hooks/useProtectionPostures.tsindex 4abae6a8f6..6c66bf0e2e 100644--- a/frontend-modern/src/hooks/useProtectionPostures.ts+++ b/frontend-modern/src/hooks/useProtectionPostures.ts@@ -10,6 +10,9 @@ import { parseProtectionPostureBatches } from '@/utils/protectionPostureResponseconst PROTECTION_POSTURES_URL = '/api/recovery/postures';export const MAX_PROTECTION_POSTURE_BATCH_SIZE = 200;+// Bound work per consuming surface as well as per request. A fleet-sized read+// must not enqueue every batch at once against the same server.+export const MAX_CONCURRENT_PROTECTION_POSTURE_REQUESTS = 4;const REFRESH_MS = 30_000;const EMPTY_POLICY: ProtectionPosturePolicy = {@@ -72,14 +75,38 @@ async function fetchAllProtectionPostures(const controller = new AbortController();const cancel = () => controller.abort(signal.reason);signal.addEventListener('abort', cancel, { once: true });+ const responses: unknown[] = new Array(batches.length);+ let nextBatch = 0;+ let failure: { error: unknown } | undefined;+ const readBatches = async () => {+ try {+ while (nextBatch < batches.length) {+ // A cancelled transport may still resolve. Never let that completion+ // dispatch a queued request for the retired scope/read.+ controller.signal.throwIfAborted();+ const index = nextBatch++;+ responses[index] = await fetchProtectionPostures(batches[index], controller.signal);+ }+ } catch (error) {+ // Stop queued work before siblings settle their aborts, and retain the+ // initiating failure rather than whichever abort rejection arrives first.+ failure ??= { error };+ controller.abort();+ throw failure.error;+ }+ };try {- const responses = await Promise.all(- batches.map((batch) => fetchProtectionPostures(batch, controller.signal)),+ await Promise.all(+ Array.from(+ { length: Math.min(MAX_CONCURRENT_PROTECTION_POSTURE_REQUESTS, batches.length) },+ readBatches,+ ),);+ // Completion order never changes request order or publishes partial health.return parseProtectionPostureBatches(responses);} catch (error) {controller.abort();- throw error;+ throw failure ? failure.error : error;} finally {signal.removeEventListener('abort', cancel);}