Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

Confirm live cluster agent connections instead of saved links

pulse-triage[bot]8be86746a9d2
diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
index 598722b1e5..f2a03023c4 100644
--- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
+++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md
@@ -26,7 +26,7 @@ closing an accepted run still does not cancel it. This is keyboard continuity,
not a change to job admission, token scope or native installation acceptance.
`frontend-modern/src/components/Settings/__tests__/ClusterAgentDeployDialog.test.tsx`
pins delayed candidate responses, safe fallbacks and both successful and failed
-terminal projections alongside the existing installed-only-after-reporting flow.
+terminal projections alongside the live-connection confirmation flow.
### Source-record lookup preserves report admission
@@ -6452,8 +6452,15 @@ cluster, member, and coverage-summary install actions open
server mints a per-node bootstrap credential for every target, so the dialog
never renders, copies, or reuses a token and the operator never pastes one.
The dialog folds target statuses into checking, installing, connecting,
-reporting, and failed, and it reports a node as installed only after the
-candidates projection shows that node with an agent. A finished deploy job
+connected, and failed. Its final badge says "Connected to Pulse", not "Reporting":
+the candidates projection must match the captured cluster ID and show both
+that exact node's saved agent link and a non-empty online source-agent entry
+for the same node. A saved link, a sibling's connection, or a finished job alone
+cannot trigger the installed callback. Missing/offline connection evidence
+remains connecting until the existing 90-second confirmation window expires,
+then says that Pulse has not confirmed a connected agent on the node and
+preserves the scoped installer fallback. This confirms connectivity, not fresh
+metrics, native installation or sustained reporting. A finished deploy job
alone is not success, because targets can still be enrolling. Clusters with no
connected agent, members Pulse has no address for, and failed targets hand off
to the scoped manual installer (`/settings/infrastructure?add=linux-host`)
diff --git a/frontend-modern/src/components/Settings/ClusterAgentDeployDialog.tsx b/frontend-modern/src/components/Settings/ClusterAgentDeployDialog.tsx
index 369b60e1fe..4612c896b6 100644
--- a/frontend-modern/src/components/Settings/ClusterAgentDeployDialog.tsx
+++ b/frontend-modern/src/components/Settings/ClusterAgentDeployDialog.tsx
@@ -25,7 +25,7 @@ import {
*/
export type ClusterAgentDeployNodeState =
- 'idle' | 'checking' | 'installing' | 'connecting' | 'reporting' | 'failed';
+ 'idle' | 'checking' | 'installing' | 'connecting' | 'connected' | 'failed';
export interface ClusterAgentDeployNodeProgress {
state: ClusterAgentDeployNodeState;
@@ -74,8 +74,8 @@ const NODE_STATE_PRESENTATION: Record<
label: 'Connecting to Pulse',
badgeClass: 'bg-blue-100 text-blue-800 dark:bg-blue-900/25 dark:text-blue-200',
},
- reporting: {
- label: 'Reporting',
+ connected: {
+ label: 'Connected to Pulse',
badgeClass: 'bg-green-100 text-green-800 dark:bg-green-900/25 dark:text-green-300',
},
failed: {
@@ -94,7 +94,7 @@ export const clusterDeployUnavailableReason = (node: ClusterDeployCandidateNode)
type Phase = 'loading' | 'select' | 'running' | 'done' | 'unavailable' | 'error';
const POLL_INTERVAL_MS = 2000;
-const REPORTING_TIMEOUT_MS = 90_000;
+const CONNECTION_TIMEOUT_MS = 90_000;
const JOB_TIMEOUT_MS = 20 * 60_000;
const REQUEST_TIMEOUT_MS = 30_000;
@@ -107,7 +107,7 @@ interface ClusterAgentDeployDialogProps {
onClose: () => void;
/** Hands off to the scoped manual installer. */
onUseInstaller: () => void;
- /** Called once at least one node is reporting, so the caller can refresh. */
+ /** Called once at least one node has a connected agent, so the caller can refresh. */
onInstalled?: () => void;
/** Overridable for tests. */
pollIntervalMs?: number;
@@ -115,6 +115,8 @@ interface ClusterAgentDeployDialogProps {
jobTimeoutMs?: number;
/** Overridable for tests. */
requestTimeoutMs?: number;
+ /** Overridable for tests. */
+ connectionTimeoutMs?: number;
}
const errorMessage = (error: unknown, fallback: string): string =>
@@ -197,6 +199,7 @@ export const ClusterAgentDeployDialog: Component<ClusterAgentDeployDialogProps>
pollMs: number;
jobTimeoutMs: number;
requestTimeoutMs: number;
+ connectionTimeoutMs: number;
onInstalled?: () => void;
}
@@ -245,11 +248,12 @@ export const ClusterAgentDeployDialog: Component<ClusterAgentDeployDialogProps>
}
};
- // A finished job only means the agent was installed and started enrolling.
- // The outcome the operator wants is Pulse counting the node, so confirm it.
- const confirmReporting = async (cfg: RunConfig, nodeIds: string[]) => {
+ // A saved node/agent link survives disconnection. Require the same node's
+ // current connection as well, not just a finished job or the saved link.
+ // This confirms connectivity, not fresh monitoring data from the node.
+ const confirmConnection = async (cfg: RunConfig, nodeIds: string[]) => {
const pending = new Set(nodeIds);
- const deadline = Date.now() + REPORTING_TIMEOUT_MS;
+ const deadline = Date.now() + cfg.connectionTimeoutMs;
while (pending.size > 0 && Date.now() < deadline) {
await wait(cfg.pollMs);
if (cfg.run !== runId) return;
@@ -258,17 +262,25 @@ export const ClusterAgentDeployDialog: Component<ClusterAgentDeployDialogProps>
ClusterAgentDeployAPI.getCandidates(cfg.clusterName),
);
if (cfg.run !== runId) return;
+ if (candidates.clusterId !== cfg.clusterName) {
+ throw new Error('Pulse could not confirm this cluster’s agent connections.');
+ }
+ const connectedNodes = new Set(
+ (candidates.sourceAgents ?? [])
+ .filter((agent) => agent.online === true && agent.agentId.trim())
+ .map((agent) => agent.nodeId),
+ );
for (const node of candidates.nodes ?? []) {
- if (pending.has(node.nodeId) && node.hasAgent) {
+ if (pending.has(node.nodeId) && node.hasAgent && connectedNodes.has(node.nodeId)) {
pending.delete(node.nodeId);
- setNodeProgress(node.nodeId, { state: 'reporting' });
+ setNodeProgress(node.nodeId, { state: 'connected' });
}
}
}
for (const nodeId of pending) {
setNodeProgress(nodeId, {
state: 'failed',
- detail: 'The agent was installed but has not reported to Pulse yet.',
+ detail: 'Pulse has not confirmed a connected agent on this node yet.',
});
}
};
@@ -281,6 +293,7 @@ export const ClusterAgentDeployDialog: Component<ClusterAgentDeployDialogProps>
pollMs: props.pollIntervalMs ?? POLL_INTERVAL_MS,
jobTimeoutMs: props.jobTimeoutMs ?? JOB_TIMEOUT_MS,
requestTimeoutMs: props.requestTimeoutMs ?? REQUEST_TIMEOUT_MS,
+ connectionTimeoutMs: props.connectionTimeoutMs ?? CONNECTION_TIMEOUT_MS,
onInstalled: props.onInstalled,
};
const targetIds = deployableNodes()
@@ -328,12 +341,12 @@ export const ClusterAgentDeployDialog: Component<ClusterAgentDeployDialogProps>
.map((target) => target.nodeId),
);
}
- if (toConfirm.length > 0) await confirmReporting(cfg, toConfirm);
+ if (toConfirm.length > 0) await confirmConnection(cfg, toConfirm);
} catch (error) {
if (cfg.run !== runId) return;
setFailure(errorMessage(error, 'The install could not be started.'));
for (const nodeId of targetIds) {
- if (progress()[nodeId]?.state !== 'reporting') {
+ if (progress()[nodeId]?.state !== 'connected') {
setNodeProgress(nodeId, { state: 'failed' });
}
}
@@ -352,7 +365,7 @@ export const ClusterAgentDeployDialog: Component<ClusterAgentDeployDialogProps>
}
}
setPhase('done');
- if (Object.values(progress()).some((entry) => entry.state === 'reporting')) {
+ if (Object.values(progress()).some((entry) => entry.state === 'connected')) {
cfg.onInstalled?.();
}
};
@@ -428,7 +441,7 @@ export const ClusterAgentDeployDialog: Component<ClusterAgentDeployDialogProps>
<For each={uncoveredNodes()}>
{(node) => (
<li class="rounded-md border border-border px-3 py-2">
- <div class="flex items-center justify-between gap-3">
+ <div class="flex flex-wrap items-center justify-between gap-3">
<label class="flex min-w-0 items-center gap-2 text-sm text-base-content">
<input
type="checkbox"
diff --git a/frontend-modern/src/components/Settings/__tests__/ClusterAgentDeployDialog.test.tsx b/frontend-modern/src/components/Settings/__tests__/ClusterAgentDeployDialog.test.tsx
index f526623c81..2b8ef840c1 100644
--- a/frontend-modern/src/components/Settings/__tests__/ClusterAgentDeployDialog.test.tsx
+++ b/frontend-modern/src/components/Settings/__tests__/ClusterAgentDeployDialog.test.tsx
@@ -39,6 +39,17 @@ const candidates = (overrides: Partial<ClusterDeployCandidates> = {}): ClusterDe
...overrides,
});
+const connectedCandidates = (): ClusterDeployCandidates =>
+ candidates({
+ sourceAgents: [
+ ...candidates().sourceAgents!,
+ { agentId: 'agent-delly2', nodeId: 'n-delly2', online: true },
+ ],
+ nodes: candidates().nodes!.map((node) =>
+ node.nodeId === 'n-delly2' ? { ...node, hasAgent: true, deployable: false } : node,
+ ),
+ });
+
const job = (
id: string,
status: ClusterDeployJob['status'],
@@ -120,13 +131,9 @@ describe('ClusterAgentDeployDialog', () => {
async (outcome) => {
const check = deferred<ClusterDeployJob>();
const completion = deferred<ClusterDeployJob>();
- api.getCandidates.mockResolvedValueOnce(candidates()).mockResolvedValue(
- candidates({
- nodes: candidates().nodes!.map((node) =>
- node.nodeId === 'n-delly2' ? { ...node, hasAgent: true, deployable: false } : node,
- ),
- }),
- );
+ api.getCandidates
+ .mockResolvedValueOnce(candidates())
+ .mockResolvedValue(connectedCandidates());
api.createPreflight.mockResolvedValue({ preflightId: 'focus-pf', status: 'running' });
api.getPreflight.mockReturnValue(check.promise);
api.createJob.mockResolvedValue({
@@ -168,7 +175,7 @@ describe('ClusterAgentDeployDialog', () => {
},
]),
);
- await screen.findByText(outcome === 'succeeded' ? 'Reporting' : 'Failed');
+ await screen.findByText(outcome === 'succeeded' ? 'Connected to Pulse' : 'Failed');
expect(screen.getByRole('button', { name: 'Close' })).toBe(dismiss);
expect(dismiss).toHaveFocus();
expect(api.createJob).toHaveBeenCalledTimes(1);
@@ -215,14 +222,8 @@ describe('ClusterAgentDeployDialog', () => {
expect(screen.getByRole('button', { name: 'Install on 1 node' })).toBeEnabled();
});
- it('checks, installs, and confirms the node is reporting before calling it done', async () => {
- api.getCandidates.mockResolvedValueOnce(candidates()).mockResolvedValue(
- candidates({
- nodes: candidates().nodes!.map((node) =>
- node.nodeId === 'n-delly2' ? { ...node, hasAgent: true, deployable: false } : node,
- ),
- }),
- );
+ it('checks, installs, and confirms the node has a connected agent before calling it done', async () => {
+ api.getCandidates.mockResolvedValueOnce(candidates()).mockResolvedValue(connectedCandidates());
api.createPreflight.mockResolvedValue({ preflightId: 'pf_1', status: 'running' });
api.getPreflight.mockResolvedValue(
job('pf_1', 'succeeded', [
@@ -255,7 +256,7 @@ describe('ClusterAgentDeployDialog', () => {
fireEvent.click(await screen.findByRole('button', { name: 'Install on 1 node' }));
- await waitFor(() => expect(screen.getByText('Reporting')).toBeInTheDocument());
+ await waitFor(() => expect(screen.getByText('Connected to Pulse')).toBeInTheDocument());
expect(api.createPreflight).toHaveBeenCalledWith('homelab', 'agent-delly', ['n-delly2']);
expect(api.createJob).toHaveBeenCalledWith('homelab', 'agent-delly', 'pf_1', ['n-delly2']);
expect(onInstalled).toHaveBeenCalledTimes(1);
@@ -263,11 +264,7 @@ describe('ClusterAgentDeployDialog', () => {
});
it('closes out a node the job never reported on instead of leaving it busy', async () => {
- const reported = candidates({
- nodes: candidates().nodes!.map((node) =>
- node.nodeId === 'n-delly2' ? { ...node, hasAgent: true, deployable: false } : node,
- ),
- });
+ const reported = connectedCandidates();
api.getCandidates.mockResolvedValueOnce(candidates()).mockResolvedValue(reported);
api.createPreflight.mockResolvedValue({ preflightId: 'pf_3', status: 'running' });
api.getPreflight.mockResolvedValue(
@@ -311,7 +308,7 @@ describe('ClusterAgentDeployDialog', () => {
expect(
await screen.findByText('Pulse could not finish installing on this node.'),
).toBeInTheDocument();
- expect(screen.getByText('Reporting')).toBeInTheDocument();
+ expect(screen.getByText('Connected to Pulse')).toBeInTheDocument();
expect(screen.getByText('Failed')).toBeInTheDocument();
expect(screen.getByRole('button', { name: 'Use the installer instead' })).toBeInTheDocument();
});
@@ -373,11 +370,7 @@ describe('ClusterAgentDeployDialog edge cases', () => {
});
it('confirms a node that gained an agent during the check without installing it', async () => {
- const covered = candidates({
- nodes: candidates().nodes!.map((node) =>
- node.nodeId === 'n-delly2' ? { ...node, hasAgent: true, deployable: false } : node,
- ),
- });
+ const covered = connectedCandidates();
api.getCandidates.mockResolvedValueOnce(candidates()).mockResolvedValue(covered);
api.createPreflight.mockResolvedValue({ preflightId: 'pf_4', status: 'running' });
api.getPreflight.mockResolvedValue(
@@ -395,11 +388,139 @@ describe('ClusterAgentDeployDialog edge cases', () => {
fireEvent.click(await screen.findByRole('button', { name: 'Install on 1 node' }));
- await waitFor(() => expect(screen.getByText('Reporting')).toBeInTheDocument());
+ await waitFor(() => expect(screen.getByText('Connected to Pulse')).toBeInTheDocument());
expect(api.createJob).not.toHaveBeenCalled();
expect(onInstalled).toHaveBeenCalledTimes(1);
});
+ it.each([
+ ['missing connections', { sourceAgents: null }],
+ ['empty connections', { sourceAgents: [] }],
+ ['only a connected sibling', { sourceAgents: candidates().sourceAgents }],
+ [
+ 'offline target',
+ { sourceAgents: [{ agentId: 'agent-delly2', nodeId: 'n-delly2', online: false }] },
+ ],
+ [
+ 'empty agent identity',
+ { sourceAgents: [{ agentId: ' ', nodeId: 'n-delly2', online: true }] },
+ ],
+ ['missing node inventory', { nodes: null }],
+ ['no saved agent link', { nodes: candidates().nodes }],
+ ] satisfies [string, Partial<ClusterDeployCandidates>][])(
+ 'does not confirm success from %s',
+ async (_name, override) => {
+ api.getCandidates.mockResolvedValueOnce(candidates()).mockResolvedValue({
+ ...connectedCandidates(),
+ ...override,
+ });
+ api.createPreflight.mockResolvedValue({ preflightId: 'pf_link', status: 'running' });
+ api.getPreflight.mockResolvedValue(
+ job('pf_link', 'succeeded', [
+ {
+ id: 't_link',
+ nodeId: 'n-delly2',
+ nodeName: 'delly2',
+ nodeIP: '192.168.0.111',
+ status: 'skipped_already_agent',
+ },
+ ]),
+ );
+ const { onInstalled } = renderDialog({
+ preselectNodeName: 'delly2',
+ connectionTimeoutMs: 100,
+ });
+ fireEvent.click(await screen.findByRole('button', { name: 'Install on 1 node' }));
+
+ await screen.findByText('Pulse has not confirmed a connected agent on this node yet.');
+ expect(screen.queryByText('Connected to Pulse')).toBeNull();
+ expect(screen.getByText('Failed')).toBeInTheDocument();
+ expect(onInstalled).not.toHaveBeenCalled();
+ expect(api.createPreflight).toHaveBeenCalledTimes(1);
+ expect(api.createJob).not.toHaveBeenCalled();
+ expect(screen.getByRole('button', { name: 'Use the installer instead' })).toBeInTheDocument();
+ },
+ );
+
+ it('waits for the installed node’s live connection, not just its saved link or successful job', async () => {
+ const live = deferred<ClusterDeployCandidates>();
+ api.getCandidates
+ .mockResolvedValueOnce(candidates())
+ .mockResolvedValueOnce({
+ ...connectedCandidates(),
+ sourceAgents: candidates().sourceAgents,
+ })
+ .mockReturnValue(live.promise);
+ api.createPreflight.mockResolvedValue({ preflightId: 'pf_live', status: 'running' });
+ api.getPreflight.mockResolvedValue(
+ job('pf_live', 'succeeded', [
+ {
+ id: 't_check',
+ nodeId: 'n-delly2',
+ nodeName: 'delly2',
+ nodeIP: '192.168.0.111',
+ status: 'ready',
+ },
+ ]),
+ );
+ api.createJob.mockResolvedValue({
+ jobId: 'dep_live',
+ acceptedTargets: ['n-delly2'],
+ skippedTargets: [],
+ });
+ api.getJob.mockResolvedValue(
+ job('dep_live', 'succeeded', [
+ {
+ id: 't_install',
+ nodeId: 'n-delly2',
+ nodeName: 'delly2',
+ nodeIP: '192.168.0.111',
+ status: 'succeeded',
+ },
+ ]),
+ );
+ const { onInstalled } = renderDialog({ preselectNodeName: 'delly2' });
+ fireEvent.click(await screen.findByRole('button', { name: 'Install on 1 node' }));
+
+ await waitFor(() => expect(api.getCandidates).toHaveBeenCalledTimes(3));
+ expect(screen.getByText('Connecting to Pulse')).toBeInTheDocument();
+ expect(screen.queryByText('Connected to Pulse')).toBeNull();
+ expect(onInstalled).not.toHaveBeenCalled();
+ live.resolve(connectedCandidates());
+ await screen.findByText('Connected to Pulse');
+ expect(onInstalled).toHaveBeenCalledTimes(1);
+ expect(api.createJob).toHaveBeenCalledTimes(1);
+ expect(api.createPreflight).toHaveBeenCalledTimes(1);
+ });
+
+ it('refuses connection confirmation from a different cluster response', async () => {
+ api.getCandidates.mockResolvedValueOnce(candidates()).mockResolvedValue({
+ ...connectedCandidates(),
+ clusterId: 'other-site',
+ });
+ api.createPreflight.mockResolvedValue({ preflightId: 'pf_other', status: 'running' });
+ api.getPreflight.mockResolvedValue(
+ job('pf_other', 'succeeded', [
+ {
+ id: 't_other',
+ nodeId: 'n-delly2',
+ nodeName: 'delly2',
+ nodeIP: '192.168.0.111',
+ status: 'skipped_already_agent',
+ },
+ ]),
+ );
+ const { onInstalled } = renderDialog({ preselectNodeName: 'delly2' });
+ fireEvent.click(await screen.findByRole('button', { name: 'Install on 1 node' }));
+
+ expect(await screen.findByRole('alert')).toHaveTextContent(
+ 'Pulse could not confirm this cluster’s agent connections.',
+ );
+ expect(screen.queryByText('Connected to Pulse')).toBeNull();
+ expect(onInstalled).not.toHaveBeenCalled();
+ expect(api.createJob).not.toHaveBeenCalled();
+ });
+
it('gives up on a stalled install with an explanation and the installer', async () => {
api.getCandidates.mockResolvedValue(candidates());
api.createPreflight.mockResolvedValue({ preflightId: 'pf_5', status: 'running' });