Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Bound fleet protection assessment request fan-out

pulse-triage[bot]afc29c6eff39
diff --git a/docs/release-control/v6/internal/subsystems/storage-recovery.md b/docs/release-control/v6/internal/subsystems/storage-recovery.md
index 6ab8ac41d0..44ea88b769 100644
--- a/docs/release-control/v6/internal/subsystems/storage-recovery.md
+++ b/docs/release-control/v6/internal/subsystems/storage-recovery.md
@@ -21,6 +21,27 @@
## Purpose
+### Fleet protection reads bound their request fan-out
+
+Coverage reads keep the existing 200-subject batch size and issue at most four
+posture requests at once per consuming surface. Queued batches start only as a
+slot completes, in request order; responses retain that order regardless of
+completion order. The entire validated fleet result and policy publish together,
+never progressively green partial health. A failed batch aborts active siblings,
+discards queued work and retains the initiating failure. A retired read cannot
+start queued batches even when a transport resolves after cancellation. Scope,
+organisation, replacement-read and disposal ownership remain unchanged, as do
+existing polling, auth, policy and independent restore-evidence inspection.
+
+Normal-suite `useProtectionPostures.concurrency.test.ts` checks peak request
+counts for a 10,001-subject fleet, out-of-order completion, atomic publication,
+failure/retry and cancellation with queued work. Existing hook/cancellation and
+Backups consumer suites remain required. Desktop/narrow browser proof must
+exercise the actual HTTP producer through Coverage for multi-wave pending,
+failure, retry and scope retirement, retaining independent evidence and focus.
+This bounds client request pressure, not measured installed CPU relief or
+original backup, guest or restore recovery.
+
### Backup inventory identities must be unambiguous
Each PVE task, archive and guest-snapshot collection, and the PBS snapshot
diff --git a/frontend-modern/src/hooks/__tests__/useProtectionPostures.concurrency.test.ts b/frontend-modern/src/hooks/__tests__/useProtectionPostures.concurrency.test.ts
new file mode 100644
index 0000000000..be2b16f30b
--- /dev/null
+++ b/frontend-modern/src/hooks/__tests__/useProtectionPostures.concurrency.test.ts
@@ -0,0 +1,194 @@
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { createRoot, createSignal } from 'solid-js';
+import { resetCreateNonSuspendingQueryCacheForTest } from '@/hooks/createNonSuspendingQuery';
+import {
+ MAX_CONCURRENT_PROTECTION_POSTURE_REQUESTS,
+ useProtectionPostures,
+} from '@/hooks/useProtectionPostures';
+import { eventBus } from '@/stores/events';
+import type { ProtectionPosturesResponse } from '@/types/recovery';
+import { protectionPolicy, protectionPosture } from '@/utils/__fixtures__/protectionPosture';
+
+const apiFetchJSONMock = vi.hoisted(() => vi.fn());
+vi.mock('@/utils/apiClient', () => ({ apiFetchJSON: apiFetchJSONMock }));
+
+const fleet = (count: number) =>
+ Array.from({ length: count }, (_, index) => `vm:${String(index).padStart(5, '0')}`);
+const disposers: (() => void)[] = [];
+interface PendingBatch {
+ ids: string[];
+ signal: AbortSignal;
+ settled: boolean;
+ finish: () => void;
+ reject: (reason: unknown) => void;
+}
+
+function holdBatches(abortRejects = false) {
+ const requests: PendingBatch[] = [];
+ let active = 0;
+ let peak = 0;
+ apiFetchJSONMock.mockImplementation(
+ (url: string, { signal }: { signal: AbortSignal }) =>
+ new Promise<ProtectionPosturesResponse>((resolve, reject) => {
+ const ids = new URL(url, 'https://pulse.invalid').searchParams.getAll('resourceId');
+ active += 1;
+ peak = Math.max(peak, active);
+ const settle = () => {
+ if (batch.settled) return false;
+ batch.settled = true;
+ active -= 1;
+ return true;
+ };
+ const batch: PendingBatch = {
+ ids,
+ signal,
+ settled: false,
+ finish: () => {
+ if (settle())
+ resolve({ data: ids.map(protectionPosture).reverse(), policy: protectionPolicy });
+ },
+ reject: (error) => {
+ if (settle()) reject(error);
+ },
+ };
+ requests.push(batch);
+ if (abortRejects)
+ signal.addEventListener(
+ 'abort',
+ () => batch.reject(new DOMException('Cancelled sibling', 'AbortError')),
+ { once: true },
+ );
+ }),
+ );
+ return { requests, peak: () => peak, active: () => active };
+}
+
+function mount(ids: () => readonly string[]) {
+ return createRoot((dispose) => {
+ disposers.push(dispose);
+ return useProtectionPostures(ids);
+ });
+}
+
+async function flush() {
+ // Drain a completed wave without advancing the 30-second polling clock.
+ for (let turn = 0; turn < 8; turn++) await Promise.resolve();
+}
+
+async function finishRead(requests: PendingBatch[], start = 0) {
+ let next = start;
+ while (next < requests.length) {
+ requests[next++].finish();
+ await flush();
+ }
+}
+
+afterEach(() => {
+ disposers.splice(0).forEach((dispose) => dispose());
+ apiFetchJSONMock.mockReset();
+ resetCreateNonSuspendingQueryCacheForTest();
+});
+
+describe('bounded protection posture fan-out', () => {
+ it('caps a 10,001-subject read at four active requests and publishes only the complete ordered result', async () => {
+ const ids = fleet(10_001);
+ const held = holdBatches();
+ const hook = mount(() => ids);
+ await vi.waitFor(() => expect(held.requests).toHaveLength(4));
+ expect(MAX_CONCURRENT_PROTECTION_POSTURE_REQUESTS).toBe(4);
+ expect(held.active()).toBe(4);
+ expect(hook.postures()).toEqual([]);
+ expect(hook.policy().freshnessWindowSeconds).toBe(0);
+
+ // Complete the fourth batch first: exactly one queued batch replaces it.
+ held.requests[3].finish();
+ await flush();
+ expect(held.requests).toHaveLength(5);
+ expect(held.active()).toBe(4);
+ expect(hook.postures()).toEqual([]);
+ expect(hook.resolvedOnce()).toBe(false);
+ await finishRead(held.requests);
+ await vi.waitFor(() => expect(hook.postures()).toHaveLength(10_001));
+ expect(held.peak()).toBe(4);
+ expect(held.requests).toHaveLength(51);
+ expect(held.requests.flatMap((batch) => batch.ids)).toEqual(ids);
+ expect(held.requests.every((batch) => batch.signal === held.requests[0].signal)).toBe(true);
+ expect(held.requests[0].signal.aborted).toBe(false);
+ expect(hook.postures().map((posture) => posture.subjectResourceId)).toEqual(
+ held.requests.flatMap((batch) => batch.ids.slice().reverse()),
+ );
+ expect(hook.policy()).toEqual(protectionPolicy);
+ expect(hook.response.error).toBeNull();
+ });
+
+ it('drops queued work and preserves the initiating failure instead of a sibling abort, then retries the whole fleet', async () => {
+ const ids = fleet(1_001);
+ const held = holdBatches(true);
+ const hook = mount(() => ids);
+ await vi.waitFor(() => expect(held.requests).toHaveLength(4));
+ const failure = new Error('Second batch unavailable');
+ held.requests[1].reject(failure);
+ await vi.waitFor(() => expect(hook.response.error).toBe(failure));
+ expect(held.requests).toHaveLength(4);
+ expect(held.requests.every((batch) => batch.signal.aborted)).toBe(true);
+ expect(held.active()).toBe(0);
+ expect(hook.postures()).toEqual([]);
+ expect(hook.policy().freshnessWindowSeconds).toBe(0);
+
+ const retry = hook.refetch();
+ expect(held.requests).toHaveLength(8);
+ expect(hook.response.error).toBe(failure);
+ held.requests[7].finish();
+ await flush();
+ expect(held.requests).toHaveLength(9);
+ expect(hook.postures()).toEqual([]);
+ await finishRead(held.requests, 4);
+ await retry;
+ expect(held.requests.slice(4).flatMap((batch) => batch.ids)).toEqual(ids);
+ expect(held.requests).toHaveLength(10);
+ expect(held.peak()).toBe(4);
+ expect(hook.postures()).toHaveLength(1_001);
+ expect(hook.policy()).toEqual(protectionPolicy);
+ expect(hook.response.error).toBeNull();
+ });
+
+ it.each(['scope', 'organisation', 'replacement', 'empty', 'disposal'] as const)(
+ 'never dispatches queued work after %s retirement, even if cancelled transports resolve late',
+ async (retirement) => {
+ const held = holdBatches();
+ const [ids, setIDs] = createSignal<readonly string[]>(fleet(1_001));
+ const hook = mount(ids);
+ await vi.waitFor(() => expect(held.requests).toHaveLength(4));
+ const retired = held.requests.slice();
+ let retry: Promise<ProtectionPosturesResponse> | undefined;
+ if (retirement === 'scope') setIDs(['vm:replacement']);
+ if (retirement === 'organisation') eventBus.emit('org_switched', 'fixture-other-org');
+ if (retirement === 'replacement') retry = hook.refetch();
+ if (retirement === 'empty') setIDs([]);
+ if (retirement === 'disposal') disposers.pop()!();
+ await flush();
+ expect(retired.every((batch) => batch.signal.aborted)).toBe(true);
+ const count = held.requests.length;
+ expect(count).toBe(
+ retirement === 'scope' ? 5 : retirement === 'empty' || retirement === 'disposal' ? 4 : 8,
+ );
+ for (const batch of retired) batch.finish();
+ await flush();
+ expect(held.requests).toHaveLength(count);
+ expect(hook.postures()).toEqual([]);
+ await finishRead(held.requests, 4);
+ await retry;
+ if (retirement === 'scope') {
+ expect(hook.postures().map((posture) => posture.subjectResourceId)).toEqual([
+ 'vm:replacement',
+ ]);
+ } else if (retirement === 'organisation' || retirement === 'replacement') {
+ expect(hook.postures()).toHaveLength(1_001);
+ expect(held.requests.slice(4).flatMap((batch) => batch.ids)).toEqual(fleet(1_001));
+ } else {
+ expect(hook.postures()).toEqual([]);
+ }
+ expect(hook.response.error).toBeNull();
+ },
+ );
+});
diff --git a/frontend-modern/src/hooks/useProtectionPostures.ts b/frontend-modern/src/hooks/useProtectionPostures.ts
index 4abae6a8f6..6c66bf0e2e 100644
--- a/frontend-modern/src/hooks/useProtectionPostures.ts
+++ b/frontend-modern/src/hooks/useProtectionPostures.ts
@@ -10,6 +10,9 @@ import { parseProtectionPostureBatches } from '@/utils/protectionPostureResponse
const PROTECTION_POSTURES_URL = '/api/recovery/postures';
export const MAX_PROTECTION_POSTURE_BATCH_SIZE = 200;
+// Bound work per consuming surface as well as per request. A fleet-sized read
+// must not enqueue every batch at once against the same server.
+export const MAX_CONCURRENT_PROTECTION_POSTURE_REQUESTS = 4;
const REFRESH_MS = 30_000;
const EMPTY_POLICY: ProtectionPosturePolicy = {
@@ -72,14 +75,38 @@ async function fetchAllProtectionPostures(
const controller = new AbortController();
const cancel = () => controller.abort(signal.reason);
signal.addEventListener('abort', cancel, { once: true });
+ const responses: unknown[] = new Array(batches.length);
+ let nextBatch = 0;
+ let failure: { error: unknown } | undefined;
+ const readBatches = async () => {
+ try {
+ while (nextBatch < batches.length) {
+ // A cancelled transport may still resolve. Never let that completion
+ // dispatch a queued request for the retired scope/read.
+ controller.signal.throwIfAborted();
+ const index = nextBatch++;
+ responses[index] = await fetchProtectionPostures(batches[index], controller.signal);
+ }
+ } catch (error) {
+ // Stop queued work before siblings settle their aborts, and retain the
+ // initiating failure rather than whichever abort rejection arrives first.
+ failure ??= { error };
+ controller.abort();
+ throw failure.error;
+ }
+ };
try {
- const responses = await Promise.all(
- batches.map((batch) => fetchProtectionPostures(batch, controller.signal)),
+ await Promise.all(
+ Array.from(
+ { length: Math.min(MAX_CONCURRENT_PROTECTION_POSTURE_REQUESTS, batches.length) },
+ readBatches,
+ ),
);
+ // Completion order never changes request order or publishes partial health.
return parseProtectionPostureBatches(responses);
} catch (error) {
controller.abort();
- throw error;
+ throw failure ? failure.error : error;
} finally {
signal.removeEventListener('abort', cancel);
}