Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
import type { Connection } from '@/api/connections';
import type { AgentFleetAgentDiagnostic, AgentFleetDiagnosticReason } from '@/api/agentDiagnostics';
import {
compareAgentVersions,
formatAgentVersionDisplay,
parseAgentVersion,
} from '@/utils/agentVersion';
import {
buildUnixAgentInstallCommand,
type AgentCommandPlatform,
} from '@/utils/agentInstallCommand';
import type { InfrastructureSystemRow } from './connectionsTableModel';
import { shellQuoteArg } from './infrastructureOperationsModel';
export type InfrastructureAgentUpdateTarget = {
key: string;
connection: Connection;
displayName: string;
contextLabel: string;
currentVersion?: string;
expectedVersion?: string;
installFlags: string[];
};
export type InfrastructureAgentDoctorStatus =
'healthy' | 'waiting' | 'warning' | 'critical' | 'removed' | 'unknown';
export type InfrastructureAgentDoctorTarget = Omit<
InfrastructureAgentUpdateTarget,
'connection'
> & {
connectionId: string;
connection?: Connection;
diagnostic?: AgentFleetAgentDiagnostic;
status: InfrastructureAgentDoctorStatus;
reasons: AgentFleetDiagnosticReason[];
evidence: string[];
needsUpdate: boolean;
needsCredentialRepair: boolean;
commandPlatform: AgentCommandPlatform | null;
commandBlockedReason?: string;
updaterLabel?: string;
profileLabel?: string;
profileVersionLabel?: string;
privilegeLabel?: string;
safeCollector: boolean;
safeProfileGuidance?: string;
actionRunnerPosture: string[];
actionRunnerCredentialEligible: boolean;
actionRunnerCredentialAction: 'issue' | 'rotate';
actionRunnerCredentialBlockReason?: string;
actionRunnerAgentId?: string;
actionRunnerHostname?: string;
lastSeen?: number | string | null;
source: 'diagnostics' | 'ledger-fallback' | 'removed';
};
export interface InfrastructureAgentDoctorOptions {
rows: readonly InfrastructureSystemRow[];
connections?: readonly Connection[];
diagnostics?: readonly AgentFleetAgentDiagnostic[];
diagnosticsAvailable: boolean;
targetVersion?: string | null;
scopedAgentIds?: readonly string[];
nowMs?: number;
}
const maybeAdd = (flags: Set<string>, flag: string) => {
if (flag.trim()) flags.add(flag);
};
export const normalizeAgentConnectionID = (value: string | null | undefined): string => {
const trimmed = (value || '').trim();
if (!trimmed) return '';
return trimmed.startsWith('agent:') ? trimmed : `agent:${trimmed}`;
};
export const diagnosticConnectionID = (diagnostic: AgentFleetAgentDiagnostic): string => {
const explicit = diagnostic.connectionId?.trim();
if (explicit) return explicit;
return normalizeAgentConnectionID(diagnostic.agentId || diagnostic.id);
};
const updateInstallFlagsForRow = (row: InfrastructureSystemRow): string[] => {
const flags = new Set<string>();
switch (row.ownerType) {
case 'pve':
maybeAdd(flags, '--enable-proxmox');
maybeAdd(flags, '--proxmox-type pve');
break;
case 'pbs':
maybeAdd(flags, '--enable-proxmox');
maybeAdd(flags, '--proxmox-type pbs');
break;
case 'docker':
maybeAdd(flags, '--enable-docker');
break;
case 'kubernetes':
maybeAdd(flags, '--enable-kubernetes');
break;
}
return Array.from(flags);
};
const connectionDisplayName = (connection: Connection): string =>
connection.agentIdentity?.hostname?.trim() ||
connection.name?.trim() ||
connection.address?.trim() ||
connection.id;
const rowContextLabel = (row: InfrastructureSystemRow): string => {
if (row.isCluster && row.name.trim()) return row.name;
if (row.ownerType === 'agent') return 'Machine';
return row.name.trim() || row.connection.name || row.ownerType;
};
const connectionNeedsUpdate = (connection: Connection, targetVersion?: string | null): boolean => {
if (connection.agentUpdateAvailable) return true;
const currentVersion = connection.agentVersion?.trim();
if (!currentVersion || !targetVersion) return false;
const comparison = compareAgentVersions(currentVersion, targetVersion);
return comparison !== null && comparison < 0;
};
const expectedVersionFor = (connection: Connection, targetVersion?: string | null) =>
connection.expectedAgentVersion?.trim() || formatAgentVersionDisplay(targetVersion) || undefined;
/**
* Agent Doctor must never turn an unknown platform into an executable command.
* Pulse only publishes agent binaries for Linux, macOS, FreeBSD, and Windows.
* Legacy Linux agents report distro identifiers instead of GOOS, so unmatched
* non-empty values are Linux unless they name a known unsupported OS family.
*/
export const resolveKnownAgentCommandPlatform = (
platform?: string | null,
): AgentCommandPlatform | null => {
const normalized = platform?.trim().toLowerCase() ?? '';
if (!normalized) return null;
if (normalized.includes('windows')) return 'windows';
if (
normalized === 'darwin' ||
normalized === 'mac' ||
normalized === 'macos' ||
normalized.includes('mac os') ||
normalized.includes('os x')
) {
return 'macos';
}
if (
normalized.includes('freebsd') ||
normalized.includes('pfsense') ||
normalized.includes('opnsense')
) {
return 'freebsd';
}
for (const unsupported of [
'aix',
'android',
'beos',
'dragonfly',
'haiku',
'hurd',
'illumos',
'ios',
'js',
'netbsd',
'openbsd',
'plan9',
'solaris',
'wasip1',
]) {
if (
normalized === unsupported ||
normalized.startsWith(`${unsupported} `) ||
normalized.startsWith(`${unsupported}-`)
) {
return null;
}
}
return 'linux';
};
const pushTarget = (
targetsByID: Map<string, InfrastructureAgentUpdateTarget>,
row: InfrastructureSystemRow,
connection?: Connection,
targetVersion?: string | null,
) => {
if (
!connection ||
connection.type !== 'agent' ||
!connectionNeedsUpdate(connection, targetVersion)
)
return;
const key = connection.id;
if (targetsByID.has(key)) return;
targetsByID.set(key, {
key,
connection,
displayName: connectionDisplayName(connection),
contextLabel: rowContextLabel(row),
currentVersion: connection.agentVersion?.trim() || undefined,
expectedVersion: expectedVersionFor(connection, targetVersion),
installFlags: updateInstallFlagsForRow(row),
});
};
export const collectInfrastructureAgentUpdateTargets = (
rows: readonly InfrastructureSystemRow[],
targetVersion?: string | null,
scopedAgentIds: readonly string[] = [],
): InfrastructureAgentUpdateTarget[] => {
const targetsByID = new Map<string, InfrastructureAgentUpdateTarget>();
const scopedAgentIDSet = new Set(
scopedAgentIds.map(normalizeAgentConnectionID).filter((value) => value.length > 0),
);
for (const row of rows) {
pushTarget(targetsByID, row, row.connection, targetVersion);
for (const connection of row.attachedConnections) {
pushTarget(targetsByID, row, connection, targetVersion);
}
for (const member of row.members) {
pushTarget(targetsByID, row, member.agentConnection, targetVersion);
}
}
return Array.from(targetsByID.values())
.filter(
(target) =>
scopedAgentIDSet.size === 0 || scopedAgentIDSet.has(normalizeAgentConnectionID(target.key)),
)
.sort((left, right) => left.displayName.localeCompare(right.displayName));
};
type AgentConnectionBinding = {
connection: Connection;
displayName: string;
contextLabel: string;
installFlags: string[];
};
const collectAgentConnectionBindings = (
rows: readonly InfrastructureSystemRow[],
connections: readonly Connection[],
): Map<string, AgentConnectionBinding> => {
const bindings = new Map<string, AgentConnectionBinding>();
const add = (row: InfrastructureSystemRow | undefined, connection?: Connection) => {
if (!connection || connection.type !== 'agent' || bindings.has(connection.id)) return;
// Machines whose telemetry comes from a platform integration (vSphere,
// TrueNAS, ...) have no Pulse Agent to diagnose; keep them out of the
// doctor instead of rendering permanent "Unknown" rows.
if (connection.integrationSource) return;
bindings.set(connection.id, {
connection,
displayName: connectionDisplayName(connection),
contextLabel: row ? rowContextLabel(row) : 'Machine',
installFlags: row ? updateInstallFlagsForRow(row) : [],
});
};
for (const row of rows) {
add(row, row.connection);
for (const connection of row.attachedConnections) add(row, connection);
for (const member of row.members) add(row, member.agentConnection);
}
// Rows deliberately suppress some duplicate physical hosts. The raw ledger
// remains authoritative for fleet membership, so retain unrepresented agents.
for (const connection of connections) add(undefined, connection);
return bindings;
};
const fallbackReason = (
code: string,
severity: 'warning' | 'critical',
message: string,
evidence: string[] = [],
): AgentFleetDiagnosticReason => ({ code, severity, message, evidence });
const ledgerFallbackReasons = (
connection: Connection,
needsUpdate: boolean,
): AgentFleetDiagnosticReason[] => {
const reasons: AgentFleetDiagnosticReason[] = [];
switch (connection.state) {
case 'unauthorized':
reasons.push(
fallbackReason(
'ledger_unauthorized',
'critical',
'The agent connection is unauthorized.',
connection.stateReason ? [connection.stateReason] : [],
),
);
break;
case 'unreachable':
reasons.push(
fallbackReason(
'ledger_unreachable',
'critical',
'The agent connection is unreachable.',
connection.stateReason ? [connection.stateReason] : [],
),
);
break;
case 'stale':
reasons.push(
fallbackReason(
'ledger_stale',
'warning',
'The agent has stopped reporting recently.',
connection.stateReason ? [connection.stateReason] : [],
),
);
break;
case 'pending':
reasons.push(fallbackReason('ledger_pending', 'warning', 'The agent has not reported yet.'));
break;
case 'paused':
reasons.push(fallbackReason('ledger_paused', 'warning', 'The agent is paused.'));
break;
}
if (
connection.state !== 'unauthorized' &&
(connection.fleet?.credentialStatus === 'invalid' ||
connection.fleet?.credentialHealth?.status === 'expired')
) {
reasons.push(
fallbackReason(
'ledger_credential_invalid',
'critical',
connection.fleet?.credentialHealth?.status === 'expired'
? 'The agent credential has expired.'
: 'The agent credential is missing or no longer valid on this Pulse server.',
),
);
}
if (connection.agentUpdate?.state === 'error') {
reasons.push(
fallbackReason(
'ledger_update_error',
'warning',
'The last agent update attempt failed.',
connection.agentUpdate.lastError ? [connection.agentUpdate.lastError] : [],
),
);
}
for (const module of connection.agentModules ?? []) {
if (module.enabled && module.state !== 'running') {
reasons.push(
fallbackReason(
'ledger_module_degraded',
'warning',
`${module.name} is enabled but ${module.state}.`,
module.lastError ? [module.lastError] : [],
),
);
}
}
if (
connection.fleet?.remoteControl === 'disconnected' ||
(connection.fleet?.commandPolicy?.status === 'blocked' &&
connection.agentIdentity?.commandsEnabled)
) {
reasons.push(
fallbackReason(
'command_channel_disconnected',
'critical',
'The agent is reporting, but its command channel is not connected.',
connection.fleet?.commandPolicy?.reason ? [connection.fleet.commandPolicy.reason] : [],
),
);
}
if (needsUpdate) {
reasons.push(
fallbackReason(
'agent_version_stale',
'warning',
'This agent is behind the supported target.',
),
);
}
return reasons;
};
const fallbackStatus = (
connection: Connection,
reasons: readonly AgentFleetDiagnosticReason[],
): InfrastructureAgentDoctorStatus => {
if (reasons.some((reason) => reason.severity === 'critical')) return 'critical';
if (reasons.length > 0) return 'warning';
if (connection.state === 'active' && connection.fleet?.versionDrift === 'current') {
return 'healthy';
}
return 'unknown';
};
const evidenceFor = (
connection: Connection | undefined,
diagnostic: AgentFleetAgentDiagnostic | undefined,
): string[] => {
const evidence = new Set<string>();
if (connection?.id) evidence.add(`Connection: ${connection.id}`);
if (connection?.agentIdentity?.hostname) {
evidence.add(`Hostname: ${connection.agentIdentity.hostname}`);
}
if (connection?.agentIdentity?.platform) {
const platform = [connection.agentIdentity.platform, connection.agentIdentity.architecture]
.filter(Boolean)
.join(' / ');
evidence.add(`Platform: ${platform}`);
}
if (connection?.agentIdentity?.reportIp) {
evidence.add(`Reported IP: ${connection.agentIdentity.reportIp}`);
}
const update = connection?.agentUpdate ?? diagnostic?.agentUpdate;
if (update?.lastCheckedAt) evidence.add(`Last updater check: ${update.lastCheckedAt}`);
if (update?.lastAttemptAt) evidence.add(`Last update attempt: ${update.lastAttemptAt}`);
if (update?.lastSuccessAt) evidence.add(`Last successful update: ${update.lastSuccessAt}`);
if (diagnostic?.machineIdFingerprint) {
evidence.add(`Machine identity: ${diagnostic.machineIdFingerprint}`);
}
for (const address of diagnostic?.interfaceAddresses ?? []) {
evidence.add(`Reported interface: ${address}`);
}
for (const reason of diagnostic?.reasons ?? []) {
for (const item of reason.evidence ?? []) evidence.add(item);
}
return Array.from(evidence);
};
// A live self-update should finish well inside this window. Past it, the
// updater is treated as stalled and the manual command comes back.
const UPDATER_IN_FLIGHT_STALL_MS = 10 * 60 * 1000;
const updaterApplyingNow = (
connection: Connection,
diagnostic: AgentFleetAgentDiagnostic | undefined,
nowMs: number,
): boolean => {
const update = connection.agentUpdate ?? diagnostic?.agentUpdate;
if (update?.state?.trim().toLowerCase() !== 'updating') return false;
const startedAt = Date.parse(update.lastAttemptAt || update.lastCheckedAt || '');
// A bare/stale "updating" state is not enough evidence to hide the manual
// recovery path forever. Only a timestamped, recent attempt is in flight.
if (Number.isNaN(startedAt)) return false;
return nowMs - startedAt < UPDATER_IN_FLIGHT_STALL_MS;
};
const updaterPresentation = (
connection: Connection,
diagnostic: AgentFleetAgentDiagnostic | undefined,
needsUpdate: boolean,
): { label?: string; waiting: boolean } => {
const update = connection.agentUpdate ?? diagnostic?.agentUpdate;
if (!update) return { waiting: false };
const state = update.state?.trim().toLowerCase();
switch (state) {
case 'updating':
return { label: 'Updating automatically', waiting: needsUpdate };
case 'checking':
return {
label: update.autoUpdate ? 'Checking for an automatic update' : 'Checking for an update',
waiting: needsUpdate && update.autoUpdate,
};
case 'update-available':
return {
label: update.autoUpdate
? 'Update queued automatically'
: 'Update available. Manual action required',
waiting: needsUpdate && update.autoUpdate,
};
case 'idle':
return {
label:
needsUpdate && update.autoUpdate
? 'Waiting for the next automatic check'
: update.autoUpdate
? 'Automatic updates ready'
: 'Manual updates only',
waiting: needsUpdate && update.autoUpdate,
};
case 'disabled':
return { label: 'Automatic updates disabled', waiting: false };
case 'error':
return { label: 'Last update attempt failed', waiting: false };
default:
return state ? { label: `Updater state: ${state}`, waiting: false } : { waiting: false };
}
};
const doctorTargetFromBinding = (
binding: AgentConnectionBinding,
diagnostic: AgentFleetAgentDiagnostic | undefined,
diagnosticsAvailable: boolean,
targetVersion: string | null | undefined,
nowMs: number,
): InfrastructureAgentDoctorTarget => {
const connection = binding.connection;
const expectedVersion = expectedVersionFor(connection, targetVersion);
const needsUpdate = connectionNeedsUpdate(connection, expectedVersion);
const fallbackReasons = ledgerFallbackReasons(connection, needsUpdate);
const reasons =
diagnosticsAvailable && diagnostic
? Array.from(
new Map(
[
...fallbackReasons.filter((reason) =>
[
'command_channel_disconnected',
'ledger_credential_invalid',
'ledger_unauthorized',
].includes(reason.code),
),
...(diagnostic.reasons ?? []),
].map((reason) => [reason.code, reason]),
).values(),
)
: fallbackReasons;
const updater = updaterPresentation(connection, diagnostic, needsUpdate);
let status: InfrastructureAgentDoctorStatus =
diagnosticsAvailable && diagnostic
? diagnostic.status
: fallbackStatus(connection, fallbackReasons);
// The ledger can advance one poll ahead of diagnostics. Never hide a live
// update/error signal while the structured endpoint catches up.
const fallbackDoctorStatus = fallbackStatus(connection, fallbackReasons);
if (fallbackDoctorStatus === 'critical') {
status = 'critical';
} else if (status === 'healthy' && fallbackReasons.length > 0) {
status = 'warning';
}
const nonVersionReasons = reasons.filter((reason) => reason.code !== 'agent_version_stale');
if (
updater.waiting &&
connection.state === 'active' &&
status !== 'critical' &&
nonVersionReasons.length === 0
) {
status = 'waiting';
}
const commandPlatform = resolveKnownAgentCommandPlatform(connection.agentIdentity?.platform);
const needsCredentialRepair = reasons.some((reason) =>
[
'agent_credential_missing',
'agent_credential_expired',
'agent_exec_scope_missing',
'ledger_credential_invalid',
'ledger_unauthorized',
].includes(reason.code),
);
const hasDuplicateInstallation = reasons.some(
(reason) => reason.code === 'duplicate_host_agent_installation',
);
let commandBlockedReason: string | undefined;
if ((needsUpdate || needsCredentialRepair) && hasDuplicateInstallation) {
commandBlockedReason =
'Multiple host-installed Pulse Agents are reporting from this machine. Pulse cannot safely choose a local service. Inspect and remove or explicitly update the duplicate installation first.';
} else if (needsUpdate && !expectedVersion) {
commandBlockedReason = 'No supported target version is available, so Pulse will not guess.';
} else if (needsUpdate && !parseAgentVersion(expectedVersion)) {
commandBlockedReason = 'The reported target version is not a supported release version.';
} else if (needsUpdate && !commandPlatform) {
commandBlockedReason =
'The agent did not report a recognized platform, so Pulse will not guess an update command.';
} else if (needsUpdate && commandPlatform === 'freebsd') {
commandBlockedReason =
'Pulse cannot verify saved FreeBSD or pfSense installer state yet. Open Install on a host for the reviewed manual path instead of running a guessed update command.';
} else if (needsUpdate && updaterApplyingNow(connection, diagnostic, nowMs)) {
commandBlockedReason =
'This agent is applying an update right now. Wait for its updater result before running a manual command.';
}
const hasStructuredUpgradeAction = Boolean(
diagnostic?.repairActions?.some(
(action) => action.code === 'copy_upgrade_command' && action.supported,
),
);
if (
needsUpdate &&
diagnosticsAvailable &&
diagnostic &&
!hasStructuredUpgradeAction &&
!commandBlockedReason
) {
commandBlockedReason = 'The diagnostic service did not offer a supported update repair.';
}
const hasStructuredCredentialRepair = Boolean(
diagnostic?.repairActions?.some(
(action) => action.code === 'repair_authentication' && action.supported,
),
);
if (
needsCredentialRepair &&
diagnosticsAvailable &&
diagnostic &&
!hasStructuredCredentialRepair &&
!commandBlockedReason
) {
commandBlockedReason =
'The diagnostic service did not identify a safe local authentication repair path.';
}
const profileLabel =
diagnostic?.profileName?.trim() || diagnostic?.profileId?.trim() || undefined;
const profileVersionLabel = diagnosticProfileVersionLabel(diagnostic);
const actionRunner = actionRunnerPresentation(diagnostic, commandPlatform, connection);
return {
key: connection.id,
connectionId: connection.id,
connection,
diagnostic,
displayName: binding.displayName,
contextLabel: binding.contextLabel,
currentVersion: connection.agentVersion?.trim() || diagnostic?.version?.trim() || undefined,
expectedVersion,
installFlags: binding.installFlags,
status,
reasons,
evidence: evidenceFor(connection, diagnostic),
needsUpdate,
needsCredentialRepair,
commandPlatform,
commandBlockedReason,
updaterLabel: updater.label,
profileLabel,
profileVersionLabel,
privilegeLabel: diagnosticPrivilegeLabel(diagnostic),
...actionRunner,
lastSeen: connection.lastSeen ?? diagnostic?.lastSeen,
source: diagnosticsAvailable && diagnostic ? 'diagnostics' : 'ledger-fallback',
};
};
const DIAGNOSTIC_ONLY_STATUSES = new Set<InfrastructureAgentDoctorStatus>([
'healthy',
'warning',
'critical',
'removed',
]);
const diagnosticDoctorStatus = (
diagnostic: AgentFleetAgentDiagnostic,
): InfrastructureAgentDoctorStatus =>
DIAGNOSTIC_ONLY_STATUSES.has(diagnostic.status as InfrastructureAgentDoctorStatus)
? (diagnostic.status as InfrastructureAgentDoctorStatus)
: 'unknown';
const DIAGNOSTIC_TYPE_LABELS: Record<string, string> = {
host: 'Host',
docker: 'Docker',
kubernetes: 'Kubernetes',
};
const diagnosticContextLabel = (
diagnostic: AgentFleetAgentDiagnostic,
removed: boolean,
): string => {
const labels = (diagnostic.types ?? []).map((type) => DIAGNOSTIC_TYPE_LABELS[type] ?? type);
if (labels.length === 0) return removed ? 'Removed agent' : 'Agent';
return `${labels.join(' + ')} agent`;
};
const diagnosticProfileVersionLabel = (
diagnostic?: AgentFleetAgentDiagnostic,
): string | undefined => {
if (!diagnostic?.profileVersion) return undefined;
if (!diagnostic.deployedProfileVersion) return `Assigned v${diagnostic.profileVersion}`;
return `Expected v${diagnostic.profileVersion} · deployed v${diagnostic.deployedProfileVersion}`;
};
// Descriptive privilege line for the doctor detail row. A least-privilege
// install is an intentional hardening profile, so this label states facts
// (service user, active scoped helpers) and never implies a defect.
const diagnosticPrivilegeLabel = (diagnostic?: AgentFleetAgentDiagnostic): string | undefined => {
const privilege = diagnostic?.privilege;
if (!privilege) return undefined;
const user = privilege.serviceUser?.trim();
const process = privilege.runningAsRoot
? 'root'
: user
? `least privilege (${user})`
: 'least privilege';
const authority = privilege.commandAuthority?.trim();
const base = authority ? `${process} · commands ${authority}` : process;
const helpers: string[] = [];
if (privilege.credentialKnown) {
helpers.push(privilege.credentialExec ? 'credential grants exec' : 'monitoring credential');
}
if (privilege.typedHelper) helpers.push('typed helper configured');
if (privilege.smartctlHelper) helpers.push('SMART helper active');
if (privilege.pctHelper) helpers.push('pct helper active');
return helpers.length > 0 ? `${base} · ${helpers.join(' · ')}` : base;
};
const isSafeCollectorPrivilege = (diagnostic?: AgentFleetAgentDiagnostic): boolean => {
const privilege = diagnostic?.privilege;
return Boolean(
privilege &&
!privilege.runningAsRoot &&
privilege.commandAuthority?.trim() === 'monitoring-only' &&
privilege.credentialKnown &&
!privilege.credentialExec &&
privilege.typedHelper,
);
};
const actionRunnerPresentation = (
diagnostic: AgentFleetAgentDiagnostic | undefined,
commandPlatform: AgentCommandPlatform | null,
connection?: Connection,
removed = false,
) => {
const privilege = diagnostic?.privilege;
const safeCollector = isSafeCollectorPrivilege(diagnostic);
const agentId =
diagnostic?.agentId?.trim() || normalizeAgentConnectionID(connection?.id).slice(6);
const hostname =
diagnostic?.hostname?.trim() || connection?.agentIdentity?.hostname?.trim() || undefined;
const systemdLooking =
commandPlatform === 'linux' && Boolean(privilege?.serviceUser?.trim()) && Boolean(hostname);
const posture: string[] = [];
if (privilege) {
posture.push(
privilege.actionRunnerCredentialIssued
? `Credential issued (${privilege.actionRunnerCredentialActive ? 'active' : 'inactive'})`
: 'Credential not issued',
);
posture.push(
privilege.actionRunnerConnected
? `Runner connected${privilege.actionRunnerVersion?.trim() ? ` · ${privilege.actionRunnerVersion.trim()}` : ''}`
: 'Runner not connected',
);
const authority = [
privilege.actionRunnerRuntimeRole?.trim(),
privilege.actionRunnerCapability?.trim(),
privilege.actionRunnerBindingVersion?.trim(),
].filter(Boolean);
if (authority.length > 0) posture.push(`Authority ${authority.join(' · ')}`);
const protocols = [
privilege.actionRunnerReceiptProtocol
? `receipt v${privilege.actionRunnerReceiptProtocol}`
: '',
privilege.actionRunnerPreflightProtocol
? `preflight v${privilege.actionRunnerPreflightProtocol}`
: '',
privilege.actionRunnerDockerObservationProtocol
? `Docker observation v${privilege.actionRunnerDockerObservationProtocol}`
: '',
].filter(Boolean);
if (protocols.length > 0) posture.push(`Protocols ${protocols.join(' · ')}`);
if (privilege.actionRunnerConnectedAt) {
posture.push(`Connected at ${new Date(privilege.actionRunnerConnectedAt).toLocaleString()}`);
}
}
let safeProfileGuidance: string | undefined;
if (!safeCollector) {
safeProfileGuidance =
commandPlatform === 'linux'
? 'Safe collector profile is not confirmed. Inspect the host service and apply --least-privilege --enable-privileged-helper only on a reviewed standard Linux systemd host.'
: 'The safe collector profile is supported only on reviewed standard Linux systemd hosts. Inspect this installation. Pulse will not default or guess a migration for this platform.';
}
let blockReason: string | undefined;
if (removed) blockReason = 'Removed agents cannot enroll an action runner.';
else if (!safeCollector) blockReason = 'Confirm the safe collector profile first.';
else if (!systemdLooking)
blockReason = 'The diagnostic does not prove a standard Linux systemd installation.';
else if (privilege?.actionRunnerConnected)
blockReason = 'The action runner is already connected.';
else if (!agentId || !hostname) blockReason = 'Canonical host identity is incomplete.';
return {
safeCollector,
safeProfileGuidance,
actionRunnerPosture: posture,
actionRunnerCredentialEligible: !blockReason,
actionRunnerCredentialAction: privilege?.actionRunnerCredentialIssued
? ('rotate' as const)
: ('issue' as const),
actionRunnerCredentialBlockReason: blockReason,
actionRunnerAgentId: agentId || undefined,
actionRunnerHostname: hostname,
};
};
const ACTION_RUNNER_TOKEN_FILE = '/etc/pulse-agent-runner/token';
export const buildActionRunnerTokenFileCommand = (): string =>
`sudo bash -c ${shellQuoteArg(
[
'set -e',
`install -d -m 0700 ${shellQuoteArg(ACTION_RUNNER_TOKEN_FILE.slice(0, ACTION_RUNNER_TOKEN_FILE.lastIndexOf('/')))}`,
'umask 077',
`IFS= read -rsp 'Action runner token: ' token </dev/tty`,
`printf '\\n' >/dev/tty`,
`printf '%s\\n' "$token" > ${shellQuoteArg(ACTION_RUNNER_TOKEN_FILE)}`,
`chmod 0600 ${shellQuoteArg(ACTION_RUNNER_TOKEN_FILE)}`,
'unset token',
].join('\n'),
)}`;
export const buildActionRunnerInstallCommand = (options: {
pulseUrl: string;
agentId: string;
hostname: string;
insecure?: boolean;
customCaPath?: string;
}): string =>
buildUnixAgentInstallCommand({
baseUrl: options.pulseUrl,
insecure: options.insecure,
caCertPath: options.customCaPath,
extraArgs: [
'--update',
'--least-privilege',
'--enable-privileged-helper',
'--enable-action-runner',
`--action-token-file ${shellQuoteArg(ACTION_RUNNER_TOKEN_FILE)}`,
`--agent-id ${shellQuoteArg(options.agentId)}`,
`--hostname ${shellQuoteArg(options.hostname)}`,
],
});
type SafeCollectorCommandOptions = {
pulseUrl: string;
insecure?: boolean;
customCaPath?: string;
};
export const buildSafeCollectorInspectCommand = (options: SafeCollectorCommandOptions): string =>
buildUnixAgentInstallCommand({
baseUrl: options.pulseUrl,
insecure: options.insecure,
caCertPath: options.customCaPath,
extraArgs: ['--safe-profile-inspect'],
});
export const buildSafeCollectorApplyCommand = (options: SafeCollectorCommandOptions): string =>
buildUnixAgentInstallCommand({
baseUrl: options.pulseUrl,
insecure: options.insecure,
caCertPath: options.customCaPath,
extraArgs: ['--safe-profile-apply'],
});
const diagnosticOnlyDoctorTarget = (
diagnostic: AgentFleetAgentDiagnostic,
): InfrastructureAgentDoctorTarget => {
const removed = diagnostic.status === 'removed';
const commandPlatform = resolveKnownAgentCommandPlatform(diagnostic.platform);
const actionRunner = actionRunnerPresentation(diagnostic, commandPlatform, undefined, removed);
return {
key: `${removed ? 'removed' : 'diagnostic'}:${diagnostic.rowKey || diagnostic.id}`,
connectionId: diagnosticConnectionID(diagnostic),
diagnostic,
displayName: diagnostic.name || diagnostic.hostname || diagnostic.id,
contextLabel: diagnosticContextLabel(diagnostic, removed),
currentVersion: diagnostic.version?.trim() || undefined,
installFlags: [],
status: removed ? 'removed' : diagnosticDoctorStatus(diagnostic),
reasons: diagnostic.reasons ?? [],
evidence: evidenceFor(undefined, diagnostic),
needsUpdate: false,
needsCredentialRepair: false,
commandPlatform,
profileLabel: diagnostic.profileName?.trim() || diagnostic.profileId?.trim() || undefined,
profileVersionLabel: diagnosticProfileVersionLabel(diagnostic),
privilegeLabel: diagnosticPrivilegeLabel(diagnostic),
...actionRunner,
lastSeen: diagnostic.lastSeen,
source: removed ? 'removed' : 'diagnostics',
};
};
const DOCTOR_STATUS_RANK: Record<InfrastructureAgentDoctorStatus, number> = {
critical: 6,
warning: 5,
waiting: 4,
unknown: 3,
removed: 2,
healthy: 1,
};
export const collectInfrastructureAgentDoctorTargets = ({
rows,
connections = [],
diagnostics = [],
diagnosticsAvailable,
targetVersion,
scopedAgentIds = [],
nowMs = Date.now(),
}: InfrastructureAgentDoctorOptions): InfrastructureAgentDoctorTarget[] => {
const bindings = collectAgentConnectionBindings(rows, connections);
const diagnosticsByConnectionID = new Map(
diagnostics
.filter((diagnostic) => diagnostic.status !== 'removed')
.map((diagnostic) => [diagnosticConnectionID(diagnostic), diagnostic]),
);
const scoped = new Set(scopedAgentIds.map(normalizeAgentConnectionID).filter(Boolean));
const inScope = (connectionId: string) =>
scoped.size === 0 || scoped.has(normalizeAgentConnectionID(connectionId));
const targets = Array.from(bindings.values())
.filter((binding) => inScope(binding.connection.id))
.map((binding) =>
doctorTargetFromBinding(
binding,
diagnosticsByConnectionID.get(binding.connection.id),
diagnosticsAvailable,
targetVersion,
nowMs,
),
);
if (diagnosticsAvailable) {
// The ledger can miss agents that report only workload telemetry (for
// example Docker-only or Kubernetes-only agents). The fleet diagnostics
// endpoint still knows them, so surface every diagnostic that found no
// ledger binding instead of silently dropping it.
for (const diagnostic of diagnostics) {
if (diagnostic.status === 'removed') {
if (scoped.size === 0) targets.push(diagnosticOnlyDoctorTarget(diagnostic));
continue;
}
const connectionId = diagnosticConnectionID(diagnostic);
if (bindings.has(connectionId) || !inScope(connectionId)) continue;
targets.push(diagnosticOnlyDoctorTarget(diagnostic));
}
}
return targets.sort(
(left, right) =>
DOCTOR_STATUS_RANK[right.status] - DOCTOR_STATUS_RANK[left.status] ||
left.displayName.localeCompare(right.displayName),
);
};
export type InfrastructureAgentDoctorUninstallHandoff = {
identity: { agentId?: string; hostname?: string };
commands: { label: string; platform: AgentCommandPlatform }[];
};
const UNINSTALL_SHELL_LABEL = 'Linux / macOS / FreeBSD';
const UNINSTALL_WINDOWS_LABEL = 'Windows PowerShell';
/**
* Removed targets keep one legitimate next step: uninstalling the agent on
* the host itself. When the retained diagnostic identity resolves to a known
* platform the handoff carries that single command; when it does not, both
* host families are offered explicitly labeled rather than guessing one.
*/
export const getInfrastructureAgentDoctorUninstallHandoff = (
target: InfrastructureAgentDoctorTarget,
): InfrastructureAgentDoctorUninstallHandoff | null => {
if (target.status !== 'removed') return null;
const identity = {
agentId: target.diagnostic?.agentId?.trim() || target.diagnostic?.id?.trim() || undefined,
hostname: target.diagnostic?.hostname?.trim() || undefined,
};
if (target.commandPlatform === 'windows') {
return { identity, commands: [{ label: UNINSTALL_WINDOWS_LABEL, platform: 'windows' }] };
}
if (target.commandPlatform) {
return {
identity,
commands: [{ label: UNINSTALL_SHELL_LABEL, platform: target.commandPlatform }],
};
}
return {
identity,
commands: [
{ label: UNINSTALL_SHELL_LABEL, platform: 'linux' },
{ label: UNINSTALL_WINDOWS_LABEL, platform: 'windows' },
],
};
};
export const summarizeInfrastructureAgentDoctorTargets = (
targets: readonly InfrastructureAgentDoctorTarget[],
) => ({
total: targets.length,
healthy: targets.filter((target) => target.status === 'healthy').length,
waiting: targets.filter((target) => target.status === 'waiting').length,
warning: targets.filter((target) => target.status === 'warning').length,
critical: targets.filter((target) => target.status === 'critical').length,
unknown: targets.filter((target) => target.status === 'unknown').length,
removed: targets.filter((target) => target.status === 'removed').length,
});
const DOCTOR_REPORT_STATUS_LABEL: Record<InfrastructureAgentDoctorStatus, string> = {
healthy: 'Healthy',
waiting: 'Waiting for updater',
warning: 'Needs attention',
critical: 'Critical',
removed: 'Removed',
unknown: 'Unknown',
};
const doctorReportLastSeen = (value?: number | string | null): string | undefined => {
if (value === null || value === undefined || value === '') return undefined;
const timestamp = typeof value === 'number' ? value : Date.parse(value);
if (!Number.isFinite(timestamp)) return typeof value === 'string' ? value : undefined;
return new Date(timestamp).toISOString();
};
// Plain-text diagnosis for pasting into a support thread or GitHub issue.
// Deliberately excludes update commands: they can embed install tokens.
export const formatInfrastructureAgentDoctorReport = (
targets: readonly InfrastructureAgentDoctorTarget[],
): string => {
const counts = summarizeInfrastructureAgentDoctorTargets(targets);
const countParts = (['critical', 'warning', 'waiting', 'unknown', 'removed', 'healthy'] as const)
.filter((status) => counts[status] > 0)
.map((status) => `${counts[status]} ${DOCTOR_REPORT_STATUS_LABEL[status].toLowerCase()}`);
const lines: string[] = [
`Pulse Agent Doctor report (${counts.total} agent${counts.total === 1 ? '' : 's'}${
countParts.length > 0 ? ` · ${countParts.join(', ')}` : ''
})`,
];
for (const target of targets) {
lines.push('');
lines.push(`${target.displayName} (${target.contextLabel})`);
lines.push(` Connection ${target.connectionId}`);
lines.push(` Status ${DOCTOR_REPORT_STATUS_LABEL[target.status]}`);
if (target.currentVersion) lines.push(` Reported agent ${target.currentVersion}`);
if (target.expectedVersion) lines.push(` Supported target ${target.expectedVersion}`);
const lastSeen = doctorReportLastSeen(target.lastSeen);
if (lastSeen) lines.push(` Last seen ${lastSeen}`);
if (target.updaterLabel) lines.push(` Updater ${target.updaterLabel}`);
if (target.privilegeLabel) lines.push(` Privilege ${target.privilegeLabel}`);
lines.push(` Safe collector ${target.safeCollector ? 'confirmed' : 'not confirmed'}`);
for (const fact of target.actionRunnerPosture ?? []) lines.push(` Action runner ${fact}`);
if (target.profileLabel) {
lines.push(
` Profile ${target.profileLabel}${
target.profileVersionLabel ? ` (${target.profileVersionLabel})` : ''
}`,
);
}
if (target.reasons.length > 0) {
lines.push(' Issues');
for (const reason of target.reasons) {
lines.push(` - ${reason.message}`);
for (const item of reason.evidence ?? []) {
lines.push(` ${item}`);
}
}
}
if (target.evidence.length > 0) {
lines.push(' Identity evidence');
for (const item of target.evidence) {
lines.push(` - ${item}`);
}
}
const repairs = (target.diagnostic?.repairActions ?? []).filter(
(action) => action.code !== 'copy_upgrade_command',
);
if (repairs.length > 0) {
lines.push(' Repair actions');
for (const repair of repairs) {
lines.push(` - ${repair.label}. ${repair.description}`);
}
}
if (target.commandBlockedReason) {
lines.push(` Update command blocked. ${target.commandBlockedReason}`);
}
}
return lines.join('\n');
};