Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

38.2 KB
Raw Blame
import type { Connection } from '@/api/connections';
import type { AgentFleetAgentDiagnostic, AgentFleetDiagnosticReason } from '@/api/agentDiagnostics';
import {
  compareAgentVersions,
  formatAgentVersionDisplay,
  parseAgentVersion,
} from '@/utils/agentVersion';
import {
  buildUnixAgentInstallCommand,
  type AgentCommandPlatform,
} from '@/utils/agentInstallCommand';
import type { InfrastructureSystemRow } from './connectionsTableModel';
import { shellQuoteArg } from './infrastructureOperationsModel';

export type InfrastructureAgentUpdateTarget = {
  key: string;
  connection: Connection;
  displayName: string;
  contextLabel: string;
  currentVersion?: string;
  expectedVersion?: string;
  installFlags: string[];
};

export type InfrastructureAgentDoctorStatus =
  'healthy' | 'waiting' | 'warning' | 'critical' | 'removed' | 'unknown';

export type InfrastructureAgentDoctorTarget = Omit<
  InfrastructureAgentUpdateTarget,
  'connection'
> & {
  connectionId: string;
  connection?: Connection;
  diagnostic?: AgentFleetAgentDiagnostic;
  status: InfrastructureAgentDoctorStatus;
  reasons: AgentFleetDiagnosticReason[];
  evidence: string[];
  needsUpdate: boolean;
  needsCredentialRepair: boolean;
  commandPlatform: AgentCommandPlatform | null;
  commandBlockedReason?: string;
  updaterLabel?: string;
  profileLabel?: string;
  profileVersionLabel?: string;
  privilegeLabel?: string;
  safeCollector: boolean;
  safeProfileGuidance?: string;
  actionRunnerPosture: string[];
  actionRunnerCredentialEligible: boolean;
  actionRunnerCredentialAction: 'issue' | 'rotate';
  actionRunnerCredentialBlockReason?: string;
  actionRunnerAgentId?: string;
  actionRunnerHostname?: string;
  lastSeen?: number | string | null;
  source: 'diagnostics' | 'ledger-fallback' | 'removed';
};

export interface InfrastructureAgentDoctorOptions {
  rows: readonly InfrastructureSystemRow[];
  connections?: readonly Connection[];
  diagnostics?: readonly AgentFleetAgentDiagnostic[];
  diagnosticsAvailable: boolean;
  targetVersion?: string | null;
  scopedAgentIds?: readonly string[];
  nowMs?: number;
}

const maybeAdd = (flags: Set<string>, flag: string) => {
  if (flag.trim()) flags.add(flag);
};

export const normalizeAgentConnectionID = (value: string | null | undefined): string => {
  const trimmed = (value || '').trim();
  if (!trimmed) return '';
  return trimmed.startsWith('agent:') ? trimmed : `agent:${trimmed}`;
};

export const diagnosticConnectionID = (diagnostic: AgentFleetAgentDiagnostic): string => {
  const explicit = diagnostic.connectionId?.trim();
  if (explicit) return explicit;
  return normalizeAgentConnectionID(diagnostic.agentId || diagnostic.id);
};

const updateInstallFlagsForRow = (row: InfrastructureSystemRow): string[] => {
  const flags = new Set<string>();

  switch (row.ownerType) {
    case 'pve':
      maybeAdd(flags, '--enable-proxmox');
      maybeAdd(flags, '--proxmox-type pve');
      break;
    case 'pbs':
      maybeAdd(flags, '--enable-proxmox');
      maybeAdd(flags, '--proxmox-type pbs');
      break;
    case 'docker':
      maybeAdd(flags, '--enable-docker');
      break;
    case 'kubernetes':
      maybeAdd(flags, '--enable-kubernetes');
      break;
  }

  return Array.from(flags);
};

const connectionDisplayName = (connection: Connection): string =>
  connection.agentIdentity?.hostname?.trim() ||
  connection.name?.trim() ||
  connection.address?.trim() ||
  connection.id;

const rowContextLabel = (row: InfrastructureSystemRow): string => {
  if (row.isCluster && row.name.trim()) return row.name;
  if (row.ownerType === 'agent') return 'Machine';
  return row.name.trim() || row.connection.name || row.ownerType;
};

const connectionNeedsUpdate = (connection: Connection, targetVersion?: string | null): boolean => {
  if (connection.agentUpdateAvailable) return true;
  const currentVersion = connection.agentVersion?.trim();
  if (!currentVersion || !targetVersion) return false;
  const comparison = compareAgentVersions(currentVersion, targetVersion);
  return comparison !== null && comparison < 0;
};

const expectedVersionFor = (connection: Connection, targetVersion?: string | null) =>
  connection.expectedAgentVersion?.trim() || formatAgentVersionDisplay(targetVersion) || undefined;

/**
 * Agent Doctor must never turn an unknown platform into an executable command.
 * Pulse only publishes agent binaries for Linux, macOS, FreeBSD, and Windows.
 * Legacy Linux agents report distro identifiers instead of GOOS, so unmatched
 * non-empty values are Linux unless they name a known unsupported OS family.
 */
export const resolveKnownAgentCommandPlatform = (
  platform?: string | null,
): AgentCommandPlatform | null => {
  const normalized = platform?.trim().toLowerCase() ?? '';
  if (!normalized) return null;
  if (normalized.includes('windows')) return 'windows';
  if (
    normalized === 'darwin' ||
    normalized === 'mac' ||
    normalized === 'macos' ||
    normalized.includes('mac os') ||
    normalized.includes('os x')
  ) {
    return 'macos';
  }
  if (
    normalized.includes('freebsd') ||
    normalized.includes('pfsense') ||
    normalized.includes('opnsense')
  ) {
    return 'freebsd';
  }
  for (const unsupported of [
    'aix',
    'android',
    'beos',
    'dragonfly',
    'haiku',
    'hurd',
    'illumos',
    'ios',
    'js',
    'netbsd',
    'openbsd',
    'plan9',
    'solaris',
    'wasip1',
  ]) {
    if (
      normalized === unsupported ||
      normalized.startsWith(`${unsupported} `) ||
      normalized.startsWith(`${unsupported}-`)
    ) {
      return null;
    }
  }
  return 'linux';
};

const pushTarget = (
  targetsByID: Map<string, InfrastructureAgentUpdateTarget>,
  row: InfrastructureSystemRow,
  connection?: Connection,
  targetVersion?: string | null,
) => {
  if (
    !connection ||
    connection.type !== 'agent' ||
    !connectionNeedsUpdate(connection, targetVersion)
  )
    return;

  const key = connection.id;
  if (targetsByID.has(key)) return;

  targetsByID.set(key, {
    key,
    connection,
    displayName: connectionDisplayName(connection),
    contextLabel: rowContextLabel(row),
    currentVersion: connection.agentVersion?.trim() || undefined,
    expectedVersion: expectedVersionFor(connection, targetVersion),
    installFlags: updateInstallFlagsForRow(row),
  });
};

export const collectInfrastructureAgentUpdateTargets = (
  rows: readonly InfrastructureSystemRow[],
  targetVersion?: string | null,
  scopedAgentIds: readonly string[] = [],
): InfrastructureAgentUpdateTarget[] => {
  const targetsByID = new Map<string, InfrastructureAgentUpdateTarget>();
  const scopedAgentIDSet = new Set(
    scopedAgentIds.map(normalizeAgentConnectionID).filter((value) => value.length > 0),
  );

  for (const row of rows) {
    pushTarget(targetsByID, row, row.connection, targetVersion);
    for (const connection of row.attachedConnections) {
      pushTarget(targetsByID, row, connection, targetVersion);
    }
    for (const member of row.members) {
      pushTarget(targetsByID, row, member.agentConnection, targetVersion);
    }
  }

  return Array.from(targetsByID.values())
    .filter(
      (target) =>
        scopedAgentIDSet.size === 0 || scopedAgentIDSet.has(normalizeAgentConnectionID(target.key)),
    )
    .sort((left, right) => left.displayName.localeCompare(right.displayName));
};

type AgentConnectionBinding = {
  connection: Connection;
  displayName: string;
  contextLabel: string;
  installFlags: string[];
};

const collectAgentConnectionBindings = (
  rows: readonly InfrastructureSystemRow[],
  connections: readonly Connection[],
): Map<string, AgentConnectionBinding> => {
  const bindings = new Map<string, AgentConnectionBinding>();
  const add = (row: InfrastructureSystemRow | undefined, connection?: Connection) => {
    if (!connection || connection.type !== 'agent' || bindings.has(connection.id)) return;
    // Machines whose telemetry comes from a platform integration (vSphere,
    // TrueNAS, ...) have no Pulse Agent to diagnose; keep them out of the
    // doctor instead of rendering permanent "Unknown" rows.
    if (connection.integrationSource) return;
    bindings.set(connection.id, {
      connection,
      displayName: connectionDisplayName(connection),
      contextLabel: row ? rowContextLabel(row) : 'Machine',
      installFlags: row ? updateInstallFlagsForRow(row) : [],
    });
  };

  for (const row of rows) {
    add(row, row.connection);
    for (const connection of row.attachedConnections) add(row, connection);
    for (const member of row.members) add(row, member.agentConnection);
  }
  // Rows deliberately suppress some duplicate physical hosts. The raw ledger
  // remains authoritative for fleet membership, so retain unrepresented agents.
  for (const connection of connections) add(undefined, connection);
  return bindings;
};

const fallbackReason = (
  code: string,
  severity: 'warning' | 'critical',
  message: string,
  evidence: string[] = [],
): AgentFleetDiagnosticReason => ({ code, severity, message, evidence });

const ledgerFallbackReasons = (
  connection: Connection,
  needsUpdate: boolean,
): AgentFleetDiagnosticReason[] => {
  const reasons: AgentFleetDiagnosticReason[] = [];
  switch (connection.state) {
    case 'unauthorized':
      reasons.push(
        fallbackReason(
          'ledger_unauthorized',
          'critical',
          'The agent connection is unauthorized.',
          connection.stateReason ? [connection.stateReason] : [],
        ),
      );
      break;
    case 'unreachable':
      reasons.push(
        fallbackReason(
          'ledger_unreachable',
          'critical',
          'The agent connection is unreachable.',
          connection.stateReason ? [connection.stateReason] : [],
        ),
      );
      break;
    case 'stale':
      reasons.push(
        fallbackReason(
          'ledger_stale',
          'warning',
          'The agent has stopped reporting recently.',
          connection.stateReason ? [connection.stateReason] : [],
        ),
      );
      break;
    case 'pending':
      reasons.push(fallbackReason('ledger_pending', 'warning', 'The agent has not reported yet.'));
      break;
    case 'paused':
      reasons.push(fallbackReason('ledger_paused', 'warning', 'The agent is paused.'));
      break;
  }

  if (
    connection.state !== 'unauthorized' &&
    (connection.fleet?.credentialStatus === 'invalid' ||
      connection.fleet?.credentialHealth?.status === 'expired')
  ) {
    reasons.push(
      fallbackReason(
        'ledger_credential_invalid',
        'critical',
        connection.fleet?.credentialHealth?.status === 'expired'
          ? 'The agent credential has expired.'
          : 'The agent credential is missing or no longer valid on this Pulse server.',
      ),
    );
  }

  if (connection.agentUpdate?.state === 'error') {
    reasons.push(
      fallbackReason(
        'ledger_update_error',
        'warning',
        'The last agent update attempt failed.',
        connection.agentUpdate.lastError ? [connection.agentUpdate.lastError] : [],
      ),
    );
  }
  for (const module of connection.agentModules ?? []) {
    if (module.enabled && module.state !== 'running') {
      reasons.push(
        fallbackReason(
          'ledger_module_degraded',
          'warning',
          `${module.name} is enabled but ${module.state}.`,
          module.lastError ? [module.lastError] : [],
        ),
      );
    }
  }
  if (
    connection.fleet?.remoteControl === 'disconnected' ||
    (connection.fleet?.commandPolicy?.status === 'blocked' &&
      connection.agentIdentity?.commandsEnabled)
  ) {
    reasons.push(
      fallbackReason(
        'command_channel_disconnected',
        'critical',
        'The agent is reporting, but its command channel is not connected.',
        connection.fleet?.commandPolicy?.reason ? [connection.fleet.commandPolicy.reason] : [],
      ),
    );
  }
  if (needsUpdate) {
    reasons.push(
      fallbackReason(
        'agent_version_stale',
        'warning',
        'This agent is behind the supported target.',
      ),
    );
  }
  return reasons;
};

const fallbackStatus = (
  connection: Connection,
  reasons: readonly AgentFleetDiagnosticReason[],
): InfrastructureAgentDoctorStatus => {
  if (reasons.some((reason) => reason.severity === 'critical')) return 'critical';
  if (reasons.length > 0) return 'warning';
  if (connection.state === 'active' && connection.fleet?.versionDrift === 'current') {
    return 'healthy';
  }
  return 'unknown';
};

const evidenceFor = (
  connection: Connection | undefined,
  diagnostic: AgentFleetAgentDiagnostic | undefined,
): string[] => {
  const evidence = new Set<string>();
  if (connection?.id) evidence.add(`Connection: ${connection.id}`);
  if (connection?.agentIdentity?.hostname) {
    evidence.add(`Hostname: ${connection.agentIdentity.hostname}`);
  }
  if (connection?.agentIdentity?.platform) {
    const platform = [connection.agentIdentity.platform, connection.agentIdentity.architecture]
      .filter(Boolean)
      .join(' / ');
    evidence.add(`Platform: ${platform}`);
  }
  if (connection?.agentIdentity?.reportIp) {
    evidence.add(`Reported IP: ${connection.agentIdentity.reportIp}`);
  }
  const update = connection?.agentUpdate ?? diagnostic?.agentUpdate;
  if (update?.lastCheckedAt) evidence.add(`Last updater check: ${update.lastCheckedAt}`);
  if (update?.lastAttemptAt) evidence.add(`Last update attempt: ${update.lastAttemptAt}`);
  if (update?.lastSuccessAt) evidence.add(`Last successful update: ${update.lastSuccessAt}`);
  if (diagnostic?.machineIdFingerprint) {
    evidence.add(`Machine identity: ${diagnostic.machineIdFingerprint}`);
  }
  for (const address of diagnostic?.interfaceAddresses ?? []) {
    evidence.add(`Reported interface: ${address}`);
  }
  for (const reason of diagnostic?.reasons ?? []) {
    for (const item of reason.evidence ?? []) evidence.add(item);
  }
  return Array.from(evidence);
};

// A live self-update should finish well inside this window. Past it, the
// updater is treated as stalled and the manual command comes back.
const UPDATER_IN_FLIGHT_STALL_MS = 10 * 60 * 1000;

const updaterApplyingNow = (
  connection: Connection,
  diagnostic: AgentFleetAgentDiagnostic | undefined,
  nowMs: number,
): boolean => {
  const update = connection.agentUpdate ?? diagnostic?.agentUpdate;
  if (update?.state?.trim().toLowerCase() !== 'updating') return false;
  const startedAt = Date.parse(update.lastAttemptAt || update.lastCheckedAt || '');
  // A bare/stale "updating" state is not enough evidence to hide the manual
  // recovery path forever. Only a timestamped, recent attempt is in flight.
  if (Number.isNaN(startedAt)) return false;
  return nowMs - startedAt < UPDATER_IN_FLIGHT_STALL_MS;
};

const updaterPresentation = (
  connection: Connection,
  diagnostic: AgentFleetAgentDiagnostic | undefined,
  needsUpdate: boolean,
): { label?: string; waiting: boolean } => {
  const update = connection.agentUpdate ?? diagnostic?.agentUpdate;
  if (!update) return { waiting: false };

  const state = update.state?.trim().toLowerCase();
  switch (state) {
    case 'updating':
      return { label: 'Updating automatically', waiting: needsUpdate };
    case 'checking':
      return {
        label: update.autoUpdate ? 'Checking for an automatic update' : 'Checking for an update',
        waiting: needsUpdate && update.autoUpdate,
      };
    case 'update-available':
      return {
        label: update.autoUpdate
          ? 'Update queued automatically'
          : 'Update available. Manual action required',
        waiting: needsUpdate && update.autoUpdate,
      };
    case 'idle':
      return {
        label:
          needsUpdate && update.autoUpdate
            ? 'Waiting for the next automatic check'
            : update.autoUpdate
              ? 'Automatic updates ready'
              : 'Manual updates only',
        waiting: needsUpdate && update.autoUpdate,
      };
    case 'disabled':
      return { label: 'Automatic updates disabled', waiting: false };
    case 'error':
      return { label: 'Last update attempt failed', waiting: false };
    default:
      return state ? { label: `Updater state: ${state}`, waiting: false } : { waiting: false };
  }
};

const doctorTargetFromBinding = (
  binding: AgentConnectionBinding,
  diagnostic: AgentFleetAgentDiagnostic | undefined,
  diagnosticsAvailable: boolean,
  targetVersion: string | null | undefined,
  nowMs: number,
): InfrastructureAgentDoctorTarget => {
  const connection = binding.connection;
  const expectedVersion = expectedVersionFor(connection, targetVersion);
  const needsUpdate = connectionNeedsUpdate(connection, expectedVersion);
  const fallbackReasons = ledgerFallbackReasons(connection, needsUpdate);
  const reasons =
    diagnosticsAvailable && diagnostic
      ? Array.from(
          new Map(
            [
              ...fallbackReasons.filter((reason) =>
                [
                  'command_channel_disconnected',
                  'ledger_credential_invalid',
                  'ledger_unauthorized',
                ].includes(reason.code),
              ),
              ...(diagnostic.reasons ?? []),
            ].map((reason) => [reason.code, reason]),
          ).values(),
        )
      : fallbackReasons;
  const updater = updaterPresentation(connection, diagnostic, needsUpdate);
  let status: InfrastructureAgentDoctorStatus =
    diagnosticsAvailable && diagnostic
      ? diagnostic.status
      : fallbackStatus(connection, fallbackReasons);

  // The ledger can advance one poll ahead of diagnostics. Never hide a live
  // update/error signal while the structured endpoint catches up.
  const fallbackDoctorStatus = fallbackStatus(connection, fallbackReasons);
  if (fallbackDoctorStatus === 'critical') {
    status = 'critical';
  } else if (status === 'healthy' && fallbackReasons.length > 0) {
    status = 'warning';
  }

  const nonVersionReasons = reasons.filter((reason) => reason.code !== 'agent_version_stale');
  if (
    updater.waiting &&
    connection.state === 'active' &&
    status !== 'critical' &&
    nonVersionReasons.length === 0
  ) {
    status = 'waiting';
  }

  const commandPlatform = resolveKnownAgentCommandPlatform(connection.agentIdentity?.platform);
  const needsCredentialRepair = reasons.some((reason) =>
    [
      'agent_credential_missing',
      'agent_credential_expired',
      'agent_exec_scope_missing',
      'ledger_credential_invalid',
      'ledger_unauthorized',
    ].includes(reason.code),
  );
  const hasDuplicateInstallation = reasons.some(
    (reason) => reason.code === 'duplicate_host_agent_installation',
  );
  let commandBlockedReason: string | undefined;
  if ((needsUpdate || needsCredentialRepair) && hasDuplicateInstallation) {
    commandBlockedReason =
      'Multiple host-installed Pulse Agents are reporting from this machine. Pulse cannot safely choose a local service. Inspect and remove or explicitly update the duplicate installation first.';
  } else if (needsUpdate && !expectedVersion) {
    commandBlockedReason = 'No supported target version is available, so Pulse will not guess.';
  } else if (needsUpdate && !parseAgentVersion(expectedVersion)) {
    commandBlockedReason = 'The reported target version is not a supported release version.';
  } else if (needsUpdate && !commandPlatform) {
    commandBlockedReason =
      'The agent did not report a recognized platform, so Pulse will not guess an update command.';
  } else if (needsUpdate && commandPlatform === 'freebsd') {
    commandBlockedReason =
      'Pulse cannot verify saved FreeBSD or pfSense installer state yet. Open Install on a host for the reviewed manual path instead of running a guessed update command.';
  } else if (needsUpdate && updaterApplyingNow(connection, diagnostic, nowMs)) {
    commandBlockedReason =
      'This agent is applying an update right now. Wait for its updater result before running a manual command.';
  }

  const hasStructuredUpgradeAction = Boolean(
    diagnostic?.repairActions?.some(
      (action) => action.code === 'copy_upgrade_command' && action.supported,
    ),
  );
  if (
    needsUpdate &&
    diagnosticsAvailable &&
    diagnostic &&
    !hasStructuredUpgradeAction &&
    !commandBlockedReason
  ) {
    commandBlockedReason = 'The diagnostic service did not offer a supported update repair.';
  }

  const hasStructuredCredentialRepair = Boolean(
    diagnostic?.repairActions?.some(
      (action) => action.code === 'repair_authentication' && action.supported,
    ),
  );
  if (
    needsCredentialRepair &&
    diagnosticsAvailable &&
    diagnostic &&
    !hasStructuredCredentialRepair &&
    !commandBlockedReason
  ) {
    commandBlockedReason =
      'The diagnostic service did not identify a safe local authentication repair path.';
  }

  const profileLabel =
    diagnostic?.profileName?.trim() || diagnostic?.profileId?.trim() || undefined;
  const profileVersionLabel = diagnosticProfileVersionLabel(diagnostic);
  const actionRunner = actionRunnerPresentation(diagnostic, commandPlatform, connection);

  return {
    key: connection.id,
    connectionId: connection.id,
    connection,
    diagnostic,
    displayName: binding.displayName,
    contextLabel: binding.contextLabel,
    currentVersion: connection.agentVersion?.trim() || diagnostic?.version?.trim() || undefined,
    expectedVersion,
    installFlags: binding.installFlags,
    status,
    reasons,
    evidence: evidenceFor(connection, diagnostic),
    needsUpdate,
    needsCredentialRepair,
    commandPlatform,
    commandBlockedReason,
    updaterLabel: updater.label,
    profileLabel,
    profileVersionLabel,
    privilegeLabel: diagnosticPrivilegeLabel(diagnostic),
    ...actionRunner,
    lastSeen: connection.lastSeen ?? diagnostic?.lastSeen,
    source: diagnosticsAvailable && diagnostic ? 'diagnostics' : 'ledger-fallback',
  };
};

const DIAGNOSTIC_ONLY_STATUSES = new Set<InfrastructureAgentDoctorStatus>([
  'healthy',
  'warning',
  'critical',
  'removed',
]);

const diagnosticDoctorStatus = (
  diagnostic: AgentFleetAgentDiagnostic,
): InfrastructureAgentDoctorStatus =>
  DIAGNOSTIC_ONLY_STATUSES.has(diagnostic.status as InfrastructureAgentDoctorStatus)
    ? (diagnostic.status as InfrastructureAgentDoctorStatus)
    : 'unknown';

const DIAGNOSTIC_TYPE_LABELS: Record<string, string> = {
  host: 'Host',
  docker: 'Docker',
  kubernetes: 'Kubernetes',
};

const diagnosticContextLabel = (
  diagnostic: AgentFleetAgentDiagnostic,
  removed: boolean,
): string => {
  const labels = (diagnostic.types ?? []).map((type) => DIAGNOSTIC_TYPE_LABELS[type] ?? type);
  if (labels.length === 0) return removed ? 'Removed agent' : 'Agent';
  return `${labels.join(' + ')} agent`;
};

const diagnosticProfileVersionLabel = (
  diagnostic?: AgentFleetAgentDiagnostic,
): string | undefined => {
  if (!diagnostic?.profileVersion) return undefined;
  if (!diagnostic.deployedProfileVersion) return `Assigned v${diagnostic.profileVersion}`;
  return `Expected v${diagnostic.profileVersion} · deployed v${diagnostic.deployedProfileVersion}`;
};

// Descriptive privilege line for the doctor detail row. A least-privilege
// install is an intentional hardening profile, so this label states facts
// (service user, active scoped helpers) and never implies a defect.
const diagnosticPrivilegeLabel = (diagnostic?: AgentFleetAgentDiagnostic): string | undefined => {
  const privilege = diagnostic?.privilege;
  if (!privilege) return undefined;
  const user = privilege.serviceUser?.trim();
  const process = privilege.runningAsRoot
    ? 'root'
    : user
      ? `least privilege (${user})`
      : 'least privilege';
  const authority = privilege.commandAuthority?.trim();
  const base = authority ? `${process} · commands ${authority}` : process;
  const helpers: string[] = [];
  if (privilege.credentialKnown) {
    helpers.push(privilege.credentialExec ? 'credential grants exec' : 'monitoring credential');
  }
  if (privilege.typedHelper) helpers.push('typed helper configured');
  if (privilege.smartctlHelper) helpers.push('SMART helper active');
  if (privilege.pctHelper) helpers.push('pct helper active');
  return helpers.length > 0 ? `${base} · ${helpers.join(' · ')}` : base;
};

const isSafeCollectorPrivilege = (diagnostic?: AgentFleetAgentDiagnostic): boolean => {
  const privilege = diagnostic?.privilege;
  return Boolean(
    privilege &&
    !privilege.runningAsRoot &&
    privilege.commandAuthority?.trim() === 'monitoring-only' &&
    privilege.credentialKnown &&
    !privilege.credentialExec &&
    privilege.typedHelper,
  );
};

const actionRunnerPresentation = (
  diagnostic: AgentFleetAgentDiagnostic | undefined,
  commandPlatform: AgentCommandPlatform | null,
  connection?: Connection,
  removed = false,
) => {
  const privilege = diagnostic?.privilege;
  const safeCollector = isSafeCollectorPrivilege(diagnostic);
  const agentId =
    diagnostic?.agentId?.trim() || normalizeAgentConnectionID(connection?.id).slice(6);
  const hostname =
    diagnostic?.hostname?.trim() || connection?.agentIdentity?.hostname?.trim() || undefined;
  const systemdLooking =
    commandPlatform === 'linux' && Boolean(privilege?.serviceUser?.trim()) && Boolean(hostname);
  const posture: string[] = [];
  if (privilege) {
    posture.push(
      privilege.actionRunnerCredentialIssued
        ? `Credential issued (${privilege.actionRunnerCredentialActive ? 'active' : 'inactive'})`
        : 'Credential not issued',
    );
    posture.push(
      privilege.actionRunnerConnected
        ? `Runner connected${privilege.actionRunnerVersion?.trim() ? ` · ${privilege.actionRunnerVersion.trim()}` : ''}`
        : 'Runner not connected',
    );
    const authority = [
      privilege.actionRunnerRuntimeRole?.trim(),
      privilege.actionRunnerCapability?.trim(),
      privilege.actionRunnerBindingVersion?.trim(),
    ].filter(Boolean);
    if (authority.length > 0) posture.push(`Authority ${authority.join(' · ')}`);
    const protocols = [
      privilege.actionRunnerReceiptProtocol
        ? `receipt v${privilege.actionRunnerReceiptProtocol}`
        : '',
      privilege.actionRunnerPreflightProtocol
        ? `preflight v${privilege.actionRunnerPreflightProtocol}`
        : '',
      privilege.actionRunnerDockerObservationProtocol
        ? `Docker observation v${privilege.actionRunnerDockerObservationProtocol}`
        : '',
    ].filter(Boolean);
    if (protocols.length > 0) posture.push(`Protocols ${protocols.join(' · ')}`);
    if (privilege.actionRunnerConnectedAt) {
      posture.push(`Connected at ${new Date(privilege.actionRunnerConnectedAt).toLocaleString()}`);
    }
  }

  let safeProfileGuidance: string | undefined;
  if (!safeCollector) {
    safeProfileGuidance =
      commandPlatform === 'linux'
        ? 'Safe collector profile is not confirmed. Inspect the host service and apply --least-privilege --enable-privileged-helper only on a reviewed standard Linux systemd host.'
        : 'The safe collector profile is supported only on reviewed standard Linux systemd hosts. Inspect this installation. Pulse will not default or guess a migration for this platform.';
  }

  let blockReason: string | undefined;
  if (removed) blockReason = 'Removed agents cannot enroll an action runner.';
  else if (!safeCollector) blockReason = 'Confirm the safe collector profile first.';
  else if (!systemdLooking)
    blockReason = 'The diagnostic does not prove a standard Linux systemd installation.';
  else if (privilege?.actionRunnerConnected)
    blockReason = 'The action runner is already connected.';
  else if (!agentId || !hostname) blockReason = 'Canonical host identity is incomplete.';

  return {
    safeCollector,
    safeProfileGuidance,
    actionRunnerPosture: posture,
    actionRunnerCredentialEligible: !blockReason,
    actionRunnerCredentialAction: privilege?.actionRunnerCredentialIssued
      ? ('rotate' as const)
      : ('issue' as const),
    actionRunnerCredentialBlockReason: blockReason,
    actionRunnerAgentId: agentId || undefined,
    actionRunnerHostname: hostname,
  };
};

const ACTION_RUNNER_TOKEN_FILE = '/etc/pulse-agent-runner/token';

export const buildActionRunnerTokenFileCommand = (): string =>
  `sudo bash -c ${shellQuoteArg(
    [
      'set -e',
      `install -d -m 0700 ${shellQuoteArg(ACTION_RUNNER_TOKEN_FILE.slice(0, ACTION_RUNNER_TOKEN_FILE.lastIndexOf('/')))}`,
      'umask 077',
      `IFS= read -rsp 'Action runner token: ' token </dev/tty`,
      `printf '\\n' >/dev/tty`,
      `printf '%s\\n' "$token" > ${shellQuoteArg(ACTION_RUNNER_TOKEN_FILE)}`,
      `chmod 0600 ${shellQuoteArg(ACTION_RUNNER_TOKEN_FILE)}`,
      'unset token',
    ].join('\n'),
  )}`;

export const buildActionRunnerInstallCommand = (options: {
  pulseUrl: string;
  agentId: string;
  hostname: string;
  insecure?: boolean;
  customCaPath?: string;
}): string =>
  buildUnixAgentInstallCommand({
    baseUrl: options.pulseUrl,
    insecure: options.insecure,
    caCertPath: options.customCaPath,
    extraArgs: [
      '--update',
      '--least-privilege',
      '--enable-privileged-helper',
      '--enable-action-runner',
      `--action-token-file ${shellQuoteArg(ACTION_RUNNER_TOKEN_FILE)}`,
      `--agent-id ${shellQuoteArg(options.agentId)}`,
      `--hostname ${shellQuoteArg(options.hostname)}`,
    ],
  });

type SafeCollectorCommandOptions = {
  pulseUrl: string;
  insecure?: boolean;
  customCaPath?: string;
};

export const buildSafeCollectorInspectCommand = (options: SafeCollectorCommandOptions): string =>
  buildUnixAgentInstallCommand({
    baseUrl: options.pulseUrl,
    insecure: options.insecure,
    caCertPath: options.customCaPath,
    extraArgs: ['--safe-profile-inspect'],
  });

export const buildSafeCollectorApplyCommand = (options: SafeCollectorCommandOptions): string =>
  buildUnixAgentInstallCommand({
    baseUrl: options.pulseUrl,
    insecure: options.insecure,
    caCertPath: options.customCaPath,
    extraArgs: ['--safe-profile-apply'],
  });

const diagnosticOnlyDoctorTarget = (
  diagnostic: AgentFleetAgentDiagnostic,
): InfrastructureAgentDoctorTarget => {
  const removed = diagnostic.status === 'removed';
  const commandPlatform = resolveKnownAgentCommandPlatform(diagnostic.platform);
  const actionRunner = actionRunnerPresentation(diagnostic, commandPlatform, undefined, removed);
  return {
    key: `${removed ? 'removed' : 'diagnostic'}:${diagnostic.rowKey || diagnostic.id}`,
    connectionId: diagnosticConnectionID(diagnostic),
    diagnostic,
    displayName: diagnostic.name || diagnostic.hostname || diagnostic.id,
    contextLabel: diagnosticContextLabel(diagnostic, removed),
    currentVersion: diagnostic.version?.trim() || undefined,
    installFlags: [],
    status: removed ? 'removed' : diagnosticDoctorStatus(diagnostic),
    reasons: diagnostic.reasons ?? [],
    evidence: evidenceFor(undefined, diagnostic),
    needsUpdate: false,
    needsCredentialRepair: false,
    commandPlatform,
    profileLabel: diagnostic.profileName?.trim() || diagnostic.profileId?.trim() || undefined,
    profileVersionLabel: diagnosticProfileVersionLabel(diagnostic),
    privilegeLabel: diagnosticPrivilegeLabel(diagnostic),
    ...actionRunner,
    lastSeen: diagnostic.lastSeen,
    source: removed ? 'removed' : 'diagnostics',
  };
};

const DOCTOR_STATUS_RANK: Record<InfrastructureAgentDoctorStatus, number> = {
  critical: 6,
  warning: 5,
  waiting: 4,
  unknown: 3,
  removed: 2,
  healthy: 1,
};

export const collectInfrastructureAgentDoctorTargets = ({
  rows,
  connections = [],
  diagnostics = [],
  diagnosticsAvailable,
  targetVersion,
  scopedAgentIds = [],
  nowMs = Date.now(),
}: InfrastructureAgentDoctorOptions): InfrastructureAgentDoctorTarget[] => {
  const bindings = collectAgentConnectionBindings(rows, connections);
  const diagnosticsByConnectionID = new Map(
    diagnostics
      .filter((diagnostic) => diagnostic.status !== 'removed')
      .map((diagnostic) => [diagnosticConnectionID(diagnostic), diagnostic]),
  );
  const scoped = new Set(scopedAgentIds.map(normalizeAgentConnectionID).filter(Boolean));
  const inScope = (connectionId: string) =>
    scoped.size === 0 || scoped.has(normalizeAgentConnectionID(connectionId));

  const targets = Array.from(bindings.values())
    .filter((binding) => inScope(binding.connection.id))
    .map((binding) =>
      doctorTargetFromBinding(
        binding,
        diagnosticsByConnectionID.get(binding.connection.id),
        diagnosticsAvailable,
        targetVersion,
        nowMs,
      ),
    );

  if (diagnosticsAvailable) {
    // The ledger can miss agents that report only workload telemetry (for
    // example Docker-only or Kubernetes-only agents). The fleet diagnostics
    // endpoint still knows them, so surface every diagnostic that found no
    // ledger binding instead of silently dropping it.
    for (const diagnostic of diagnostics) {
      if (diagnostic.status === 'removed') {
        if (scoped.size === 0) targets.push(diagnosticOnlyDoctorTarget(diagnostic));
        continue;
      }
      const connectionId = diagnosticConnectionID(diagnostic);
      if (bindings.has(connectionId) || !inScope(connectionId)) continue;
      targets.push(diagnosticOnlyDoctorTarget(diagnostic));
    }
  }

  return targets.sort(
    (left, right) =>
      DOCTOR_STATUS_RANK[right.status] - DOCTOR_STATUS_RANK[left.status] ||
      left.displayName.localeCompare(right.displayName),
  );
};

export type InfrastructureAgentDoctorUninstallHandoff = {
  identity: { agentId?: string; hostname?: string };
  commands: { label: string; platform: AgentCommandPlatform }[];
};

const UNINSTALL_SHELL_LABEL = 'Linux / macOS / FreeBSD';
const UNINSTALL_WINDOWS_LABEL = 'Windows PowerShell';

/**
 * Removed targets keep one legitimate next step: uninstalling the agent on
 * the host itself. When the retained diagnostic identity resolves to a known
 * platform the handoff carries that single command; when it does not, both
 * host families are offered explicitly labeled rather than guessing one.
 */
export const getInfrastructureAgentDoctorUninstallHandoff = (
  target: InfrastructureAgentDoctorTarget,
): InfrastructureAgentDoctorUninstallHandoff | null => {
  if (target.status !== 'removed') return null;
  const identity = {
    agentId: target.diagnostic?.agentId?.trim() || target.diagnostic?.id?.trim() || undefined,
    hostname: target.diagnostic?.hostname?.trim() || undefined,
  };
  if (target.commandPlatform === 'windows') {
    return { identity, commands: [{ label: UNINSTALL_WINDOWS_LABEL, platform: 'windows' }] };
  }
  if (target.commandPlatform) {
    return {
      identity,
      commands: [{ label: UNINSTALL_SHELL_LABEL, platform: target.commandPlatform }],
    };
  }
  return {
    identity,
    commands: [
      { label: UNINSTALL_SHELL_LABEL, platform: 'linux' },
      { label: UNINSTALL_WINDOWS_LABEL, platform: 'windows' },
    ],
  };
};

export const summarizeInfrastructureAgentDoctorTargets = (
  targets: readonly InfrastructureAgentDoctorTarget[],
) => ({
  total: targets.length,
  healthy: targets.filter((target) => target.status === 'healthy').length,
  waiting: targets.filter((target) => target.status === 'waiting').length,
  warning: targets.filter((target) => target.status === 'warning').length,
  critical: targets.filter((target) => target.status === 'critical').length,
  unknown: targets.filter((target) => target.status === 'unknown').length,
  removed: targets.filter((target) => target.status === 'removed').length,
});

const DOCTOR_REPORT_STATUS_LABEL: Record<InfrastructureAgentDoctorStatus, string> = {
  healthy: 'Healthy',
  waiting: 'Waiting for updater',
  warning: 'Needs attention',
  critical: 'Critical',
  removed: 'Removed',
  unknown: 'Unknown',
};

const doctorReportLastSeen = (value?: number | string | null): string | undefined => {
  if (value === null || value === undefined || value === '') return undefined;
  const timestamp = typeof value === 'number' ? value : Date.parse(value);
  if (!Number.isFinite(timestamp)) return typeof value === 'string' ? value : undefined;
  return new Date(timestamp).toISOString();
};

// Plain-text diagnosis for pasting into a support thread or GitHub issue.
// Deliberately excludes update commands: they can embed install tokens.
export const formatInfrastructureAgentDoctorReport = (
  targets: readonly InfrastructureAgentDoctorTarget[],
): string => {
  const counts = summarizeInfrastructureAgentDoctorTargets(targets);
  const countParts = (['critical', 'warning', 'waiting', 'unknown', 'removed', 'healthy'] as const)
    .filter((status) => counts[status] > 0)
    .map((status) => `${counts[status]} ${DOCTOR_REPORT_STATUS_LABEL[status].toLowerCase()}`);
  const lines: string[] = [
    `Pulse Agent Doctor report (${counts.total} agent${counts.total === 1 ? '' : 's'}${
      countParts.length > 0 ? ` · ${countParts.join(', ')}` : ''
    })`,
  ];
  for (const target of targets) {
    lines.push('');
    lines.push(`${target.displayName} (${target.contextLabel})`);
    lines.push(`  Connection ${target.connectionId}`);
    lines.push(`  Status ${DOCTOR_REPORT_STATUS_LABEL[target.status]}`);
    if (target.currentVersion) lines.push(`  Reported agent ${target.currentVersion}`);
    if (target.expectedVersion) lines.push(`  Supported target ${target.expectedVersion}`);
    const lastSeen = doctorReportLastSeen(target.lastSeen);
    if (lastSeen) lines.push(`  Last seen ${lastSeen}`);
    if (target.updaterLabel) lines.push(`  Updater ${target.updaterLabel}`);
    if (target.privilegeLabel) lines.push(`  Privilege ${target.privilegeLabel}`);
    lines.push(`  Safe collector ${target.safeCollector ? 'confirmed' : 'not confirmed'}`);
    for (const fact of target.actionRunnerPosture ?? []) lines.push(`  Action runner ${fact}`);
    if (target.profileLabel) {
      lines.push(
        `  Profile ${target.profileLabel}${
          target.profileVersionLabel ? ` (${target.profileVersionLabel})` : ''
        }`,
      );
    }
    if (target.reasons.length > 0) {
      lines.push('  Issues');
      for (const reason of target.reasons) {
        lines.push(`  - ${reason.message}`);
        for (const item of reason.evidence ?? []) {
          lines.push(`    ${item}`);
        }
      }
    }
    if (target.evidence.length > 0) {
      lines.push('  Identity evidence');
      for (const item of target.evidence) {
        lines.push(`  - ${item}`);
      }
    }
    const repairs = (target.diagnostic?.repairActions ?? []).filter(
      (action) => action.code !== 'copy_upgrade_command',
    );
    if (repairs.length > 0) {
      lines.push('  Repair actions');
      for (const repair of repairs) {
        lines.push(`  - ${repair.label}. ${repair.description}`);
      }
    }
    if (target.commandBlockedReason) {
      lines.push(`  Update command blocked. ${target.commandBlockedReason}`);
    }
  }
  return lines.join('\n');
};