Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

19.1 KB
Raw Blame
import type {
  Disk,
  Agent,
  HostNetworkInterface,
  HostRAIDArray,
  HostSensorSummary,
  Memory,
  Node,
  PBSBackupJob,
  PBSGarbageJob,
  PBSJobHealthEvidence,
  PBSPruneJob,
  PBSSyncJob,
  PBSVerifyJob,
} from '@/types/api';
import type { Resource, ResourceMetric, ResourceVMwareMeta } from '@/types/resource';
import { formatTemperature } from '@/utils/temperature';
import { getPhysicalDiskTemperaturePresentation } from '@/features/storageBackups/diskTemperaturePresentation';
import { formatBytes } from '@/utils/format';
import { getActionableAgentIdFromResource } from '@/utils/agentResources';
import {
  getPreferredInfrastructureDisplayName,
  getPreferredResourceHostname,
} from '@/utils/resourceIdentity';
import { titleCaseDelimitedLabel } from '@/utils/textPresentation';
export { getSourceTypeLabel as formatSourceType } from '@/utils/sourceTypePresentation';

export type ProxmoxPlatformData = {
  nodeName?: string;
  clusterName?: string;
  vmid?: number;
  memory?: Partial<Memory>;
  pveVersion?: string;
  kernelVersion?: string;
  uptime?: number;
  cpuInfo?: { model?: string; cores?: number; sockets?: number };
};

export type AgentDiskInfo = {
  device?: string;
  mountpoint?: string;
  filesystem?: string;
  type?: string;
  total?: number;
  used?: number;
  free?: number;
};

export type AgentPlatformData = {
  agentId?: string;
  agentVersion?: string;
  hostname?: string;
  platform?: string;
  osName?: string;
  osVersion?: string;
  kernelVersion?: string;
  architecture?: string;
  uptimeSeconds?: number;
  networkInterfaces?: HostNetworkInterface[];
  disks?: AgentDiskInfo[];
  sensors?: HostSensorSummary;
  raid?: HostRAIDArray[];
  cpuCount?: number;
  memory?: Partial<Memory>;
  commandsEnabled?: boolean;
};

export type PBSPlatformData = {
  instanceId?: string;
  hostname?: string;
  version?: string;
  uptimeSeconds?: number;
  datastoreCount?: number;
  backupJobCount?: number;
  syncJobCount?: number;
  verifyJobCount?: number;
  pruneJobCount?: number;
  garbageJobCount?: number;
  jobHealthEvidenceCount?: number;
  backupJobs?: PBSBackupJob[];
  syncJobs?: PBSSyncJob[];
  verifyJobs?: PBSVerifyJob[];
  pruneJobs?: PBSPruneJob[];
  garbageJobs?: PBSGarbageJob[];
  jobHealthEvidence?: PBSJobHealthEvidence[];
  connectionHealth?: string;
};

export type PMGPlatformData = {
  instanceId?: string;
  hostname?: string;
  version?: string;
  nodeCount?: number;
  uptimeSeconds?: number;
  queueActive?: number;
  queueDeferred?: number;
  queueHold?: number;
  queueIncoming?: number;
  queueTotal?: number;
  mailCountTotal?: number;
  spamIn?: number;
  virusIn?: number;
  connectionHealth?: string;
  lastUpdated?: string;
};

export type KubernetesMetricCapabilities = {
  nodeCpuMemory?: boolean;
  nodeTelemetry?: boolean;
  podCpuMemory?: boolean;
  podNetwork?: boolean;
  podEphemeralDisk?: boolean;
  podDiskIo?: boolean;
};

export type KubernetesPlatformData = {
  clusterId?: string;
  agentId?: string;
  clusterName?: string;
  context?: string;
  namespace?: string;
  podName?: string;
  podUid?: string;
  metricCapabilities?: KubernetesMetricCapabilities;
};

export type PlatformData = {
  sources?: string[];
  proxmox?: ProxmoxPlatformData;
  agent?: AgentPlatformData;
  sourceStatus?: Record<string, { status?: string; lastSeen?: string | number; error?: string }>;
  docker?: Record<string, unknown>;
  vmware?: ResourceVMwareMeta;
  pbs?: PBSPlatformData;
  pmg?: PMGPlatformData;
  kubernetes?: KubernetesPlatformData;
  metrics?: Record<string, unknown>;
  identityMatch?: unknown;
  matchResults?: unknown;
  matchCandidates?: unknown;
  matches?: unknown;
};

export type DockerRuntimeCommand = {
  id?: string;
  type?: string;
  status?: string;
  message?: string;
  failureReason?: string;
  createdAt?: string;
  updatedAt?: string;
  acknowledgedAt?: string;
  completedAt?: string;
  failedAt?: string;
};

export type DockerPlatformData = {
  hostSourceId?: string;
  containerId?: string;
  hostname?: string;
  runtime?: string;
  runtimeVersion?: string;
  dockerVersion?: string;
  collectionMode?: string;
  os?: string;
  kernelVersion?: string;
  architecture?: string;
  agentVersion?: string;
  uptimeSeconds?: number;
  swarm?: {
    nodeId?: string;
    nodeRole?: string;
    localState?: string;
    controlAvailable?: boolean;
    clusterId?: string;
    clusterName?: string;
    scope?: string;
    error?: string;
  };
  containerCount?: number;
  updatesAvailableCount?: number;
  updatesLastCheckedAt?: string;
  command?: DockerRuntimeCommand;
};

const asString = (value: unknown): string | undefined =>
  typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;

const getPreferredHostLabel = (resource: Resource): string =>
  getPreferredResourceHostname(resource) ||
  getPreferredInfrastructureDisplayName(resource) ||
  resource.id;

export const buildMemory = (metric?: ResourceMetric, fallback?: Partial<Memory>): Memory => {
  const usageUnavailable = metric == null && fallback?.usageUnavailable === true;
  const total = metric?.total ?? fallback?.total ?? 0;
  const used = metric?.used ?? fallback?.used ?? 0;
  const free = metric?.free ?? fallback?.free ?? (usageUnavailable ? 0 : Math.max(total - used, 0));
  const usage = usageUnavailable ? 0 : total > 0 ? used / total : (fallback?.usage ?? 0);
  return {
    total,
    used,
    free,
    usage,
    ...(usageUnavailable ? { usageUnavailable: true } : {}),
  };
};

export const buildDisk = (metric?: ResourceMetric, fallback?: Partial<Disk>): Disk => {
  const total = metric?.total ?? fallback?.total ?? 0;
  const used = metric?.used ?? fallback?.used ?? 0;
  const free = metric?.free ?? fallback?.free ?? Math.max(total - used, 0);
  const usage = total > 0 ? used / total : (fallback?.usage ?? 0);
  return {
    total,
    used,
    free,
    usage,
    mountpoint: fallback?.mountpoint,
    type: fallback?.type,
    device: fallback?.device,
  };
};

export const toAgentDisks = (disks?: AgentDiskInfo[]): Disk[] | undefined => {
  if (!disks || disks.length === 0) return undefined;
  return disks.map((disk) => {
    const total = disk.total ?? 0;
    const used = disk.used ?? 0;
    const free = disk.free ?? Math.max(total - used, 0);
    const usage = total > 0 ? used / total : 0;
    return {
      total,
      used,
      free,
      usage,
      mountpoint: disk.mountpoint ?? disk.device,
      type: disk.filesystem ?? disk.type,
      device: disk.device,
    };
  });
};

export const toNodeFromProxmox = (resource: Resource): Node | null => {
  const platformData = resource.platformData as PlatformData | undefined;
  const proxmox = platformData?.proxmox;
  if (!proxmox) return null;

  const memory = buildMemory(resource.memory, proxmox.memory);
  const disk = buildDisk(resource.disk);
  const lastSeen = Number.isFinite(resource.lastSeen)
    ? new Date(resource.lastSeen).toISOString()
    : new Date().toISOString();
  const linkedAgentId =
    asString((platformData as { linkedAgentId?: unknown } | undefined)?.linkedAgentId) ||
    getActionableAgentIdFromResource(resource);

  return {
    id: resource.id,
    name: proxmox.nodeName ?? getPreferredHostLabel(resource),
    displayName: getPreferredInfrastructureDisplayName(resource),
    instance: resource.platformId ?? resource.id,
    host: proxmox.nodeName ?? getPreferredHostLabel(resource),
    status: resource.status,
    type: resource.type,
    cpu: resource.cpu?.current ?? 0,
    memory,
    disk,
    uptime: resource.uptime ?? proxmox.uptime ?? 0,
    loadAverage: [],
    kernelVersion: proxmox.kernelVersion ?? 'Unknown',
    pveVersion: proxmox.pveVersion ?? 'Unknown',
    cpuInfo: {
      model: proxmox.cpuInfo?.model ?? 'Unknown',
      cores: proxmox.cpuInfo?.cores ?? 0,
      sockets: proxmox.cpuInfo?.sockets ?? 0,
      mhz: '0',
    },
    lastSeen,
    connectionHealth: resource.status ?? 'unknown',
    linkedAgentId,
  } as Node;
};

export const toAgentFromResource = (
  resource: Resource,
  explicitAgent?: AgentPlatformData,
): Agent | null => {
  const platformData = resource.platformData as PlatformData | undefined;
  const agent = explicitAgent ?? platformData?.agent;
  if (!agent) return null;

  const proxmoxCores = platformData?.proxmox?.cpuInfo?.cores;
  const cpuCount = [
    agent.cpuCount,
    (agent as { cpuCores?: number }).cpuCores,
    (agent as { cores?: number }).cores,
    proxmoxCores,
  ].find((value) => typeof value === 'number' && value > 0);

  const hostname = agent.hostname ?? getPreferredHostLabel(resource);
  const agentId = getActionableAgentIdFromResource(resource) || resource.id;

  return {
    id: agentId,
    hostname,
    displayName: getPreferredInfrastructureDisplayName(resource),
    platform: agent.platform,
    osName: agent.osName ?? 'Unknown',
    osVersion: agent.osVersion ?? '',
    kernelVersion: agent.kernelVersion ?? 'Unknown',
    architecture: agent.architecture ?? 'Unknown',
    cpuCount,
    memory: buildMemory(resource.memory, agent.memory),
    disks: toAgentDisks(agent.disks),
    networkInterfaces: agent.networkInterfaces,
    sensors: agent.sensors,
    raid: agent.raid,
    status: resource.status,
    // A silent agent's uptime is from its last report, not how long the
    // machine has been up.
    uptimeSeconds: resource.agent?.stale ? 0 : (agent.uptimeSeconds ?? resource.uptime ?? 0),
    lastSeen: resource.lastSeen,
    agentVersion: agent.agentVersion,
    commandsEnabled: agent.commandsEnabled,
    tags: resource.tags,
  } as Agent;
};

export const formatSensorName = (name: string) => {
  let clean = name.replace(/^[a-z]+\d*_/i, '');
  return titleCaseDelimitedLabel(clean);
};

const formatSensorHardwareName = (name: string) => {
  return titleCaseDelimitedLabel(name)
    .replace(/\bCpu\b/g, 'CPU')
    .replace(/\bDram\b/g, 'DRAM')
    .replace(/\bGpu\b/g, 'GPU')
    .replace(/\bVram\b/g, 'VRAM')
    .replace(/\bVrm\b/g, 'VRM');
};

const formatPowerSensorLabel = (name: string) => {
  const label = formatSensorHardwareName(name);
  return /\b(power|watts?|wattage)\b/i.test(label) ? label : `${label} Power`;
};

const formatFanSensorLabel = (name: string) => {
  const label = formatSensorHardwareName(name);
  return /\bfan\b/i.test(label) ? label : `${label} Fan`;
};

const formatGPUStatsLabel = (id: string | undefined, index: number): string => {
  const trimmed = (id || '').trim();
  return trimmed ? `GPU ${trimmed}` : `GPU ${index + 1}`;
};

const formatGPUStatsValue = (gpu: NonNullable<HostSensorSummary['gpu']>[number]) => {
  const parts: string[] = [];
  if (gpu.name) parts.push(gpu.name);
  if (typeof gpu.temperatureCelsius === 'number' && Number.isFinite(gpu.temperatureCelsius)) {
    parts.push(formatTemperature(gpu.temperatureCelsius));
  }
  if (typeof gpu.utilizationPercent === 'number' && Number.isFinite(gpu.utilizationPercent)) {
    parts.push(`${Math.round(Math.max(0, gpu.utilizationPercent))}%`);
  }
  if (
    typeof gpu.memoryTotalBytes === 'number' &&
    Number.isFinite(gpu.memoryTotalBytes) &&
    gpu.memoryTotalBytes > 0
  ) {
    const used =
      typeof gpu.memoryUsedBytes === 'number' && Number.isFinite(gpu.memoryUsedBytes)
        ? Math.max(0, gpu.memoryUsedBytes)
        : 0;
    parts.push(`${formatBytes(used)} / ${formatBytes(gpu.memoryTotalBytes)}`);
  }
  return parts.join(' · ');
};

const formatPowerWatts = (value: number) => {
  if (!Number.isFinite(value)) return '';
  if (Math.abs(value) >= 100) return `${Math.round(value).toLocaleString()} W`;
  return `${value.toFixed(1)} W`;
};

export const buildPowerRows = (power?: HostSensorSummary['powerWatts']) =>
  Object.entries(power ?? {})
    .map(([name, value]) => [name, formatPowerWatts(value)] as const)
    .filter(([, value]) => value)
    .sort(([a], [b]) => a.localeCompare(b))
    .map(([name, value]) => {
      const label = formatPowerSensorLabel(name);
      return { label, value, valueTitle: `${label} ${value}` };
    });

const formatFanRPM = (value: number) => {
  if (!Number.isFinite(value)) return '';
  return `${Math.round(Math.max(0, value)).toLocaleString()} RPM`;
};

const buildTypedGPUTemperatureKeys = (gpus?: HostSensorSummary['gpu']) => {
  const keys = new Set<string>();
  gpus?.forEach((gpu) => {
    const id = (gpu.id || '').trim();
    if (!id) return;
    if (typeof gpu.temperatureCelsius !== 'number' || !Number.isFinite(gpu.temperatureCelsius))
      return;
    keys.add(`gpu_nvidia_${id}`);
  });
  return keys;
};

// Matches the backend's reason for readings a host agent past its reporting
// lease no longer collects (models.HostAgentStoppedReportingReason).
export const HOST_AGENT_STOPPED_REPORTING_REASON = 'host agent stopped reporting';

export type TemperatureRowsOptions = {
  /**
   * Why the machine's own sensor readings are retained rather than current,
   * such as a host agent past its reporting lease. Disk rows keep their own
   * collection state instead.
   */
  lastKnownReason?: string;
};

export const buildTemperatureRows = (
  sensors?: HostSensorSummary,
  options: TemperatureRowsOptions = {},
) => {
  const rows: { label: string; value: string; valueTitle?: string }[] = [];
  const thermalState = sensors?.thermalState;
  const typedGPUTemperatureKeys = buildTypedGPUTemperatureKeys(sensors?.gpu);
  if (thermalState?.pressure) {
    rows.push({
      label: 'Thermal pressure',
      value: titleCaseDelimitedLabel(thermalState.pressure),
      valueTitle: thermalState.source
        ? `${titleCaseDelimitedLabel(thermalState.pressure)} via ${thermalState.source}`
        : titleCaseDelimitedLabel(thermalState.pressure),
    });
  }
  const limits = thermalState?.limitsPercent;
  if (limits) {
    Object.entries(limits)
      .filter(([, value]) => Number.isFinite(value) && value < 100)
      .sort(([a], [b]) => a.localeCompare(b))
      .forEach(([name, value]) => {
        rows.push({
          label: formatSensorName(name),
          value: `${Math.round(value)}%`,
          valueTitle: `${formatSensorName(name)} ${Math.round(value)}%`,
        });
      });
  }
  const gpus = sensors?.gpu;
  if (gpus) {
    gpus.forEach((gpu, index) => {
      const value = formatGPUStatsValue(gpu);
      if (!value) return;
      rows.push({
        label: formatGPUStatsLabel(gpu.id, index),
        value,
        valueTitle: value,
      });
    });
  }
  const temps = sensors?.temperatureCelsius;
  if (temps) {
    const entries = Object.entries(temps)
      .filter(([name]) => !typedGPUTemperatureKeys.has(name))
      .filter(([, temp]) => Number.isFinite(temp))
      .sort(([a], [b]) => a.localeCompare(b));
    entries.forEach(([name, temp]) => {
      rows.push({
        label: formatSensorName(name),
        value: formatTemperature(temp),
        valueTitle: `${temp.toFixed(1)}°C`,
      });
    });
  }

  const additional = sensors?.additional;
  if (additional) {
    Object.entries(additional)
      .filter(([, value]) => Number.isFinite(value))
      .sort(([a], [b]) => a.localeCompare(b))
      .forEach(([name, value]) => {
        const label = formatSensorHardwareName(name);
        rows.push({
          label,
          value: formatTemperature(value),
          valueTitle: `${value.toFixed(1)}°C`,
        });
      });
  }

  const fans = sensors?.fanRpm;
  if (fans) {
    Object.entries(fans)
      .map(([name, value]) => [name, formatFanRPM(value)] as const)
      .filter(([, value]) => value)
      .sort(([a], [b]) => a.localeCompare(b))
      .forEach(([name, value]) => {
        const label = formatFanSensorLabel(name);
        rows.push({
          label,
          value,
          valueTitle: `${label} ${value}`,
        });
      });
  }

  rows.push(...buildPowerRows(sensors?.powerWatts));

  // This card has no other cue that a value is not current, so a retained
  // sensor reading says so in its value, as a retained disk reading does.
  const lastKnownReason = options.lastKnownReason?.trim();
  if (lastKnownReason) {
    const title = `Last known reading, not current: ${lastKnownReason}`;
    for (const row of rows) {
      row.value = `${row.value} (last known)`;
      row.valueTitle = title;
    }
  }

  const smart = sensors?.smart;
  if (smart) {
    smart
      .filter((disk) => !disk.standby)
      .sort((a, b) => a.device.localeCompare(b.device))
      .forEach((disk) => {
        const temperature = getPhysicalDiskTemperaturePresentation(disk);
        if (!temperature) return;
        // A retained reading, such as a silent agent's, says so in the value
        // because this card has no other cue that it is not current.
        rows.push({
          label: `Disk ${disk.device}`,
          value: temperature.current ? temperature.label : `${temperature.label} (last known)`,
          valueTitle: temperature.title ?? `${disk.temperature.toFixed(1)}°C`,
        });
      });
  }

  return rows;
};

const formatCustomSensorValue = (value: number, unit?: string): string => {
  const formatted = new Intl.NumberFormat(undefined, {
    maximumFractionDigits: 3,
  }).format(value);
  const normalizedUnit = (unit || '').trim();
  if (!normalizedUnit) return formatted;
  if (normalizedUnit === '%' || normalizedUnit.startsWith('°'))
    return `${formatted}${normalizedUnit}`;
  return `${formatted} ${normalizedUnit}`;
};

const formatCustomSensorAge = (seconds: number): string => {
  const normalized = Math.max(0, Math.floor(seconds));
  if (normalized < 60) return `${normalized}s ago`;
  if (normalized < 3600) return `${Math.floor(normalized / 60)}m ago`;
  if (normalized < 86400) return `${Math.floor(normalized / 3600)}h ago`;
  return `${Math.floor(normalized / 86400)}d ago`;
};

export const buildCustomSensorRows = (sensors?: HostSensorSummary) =>
  (sensors?.custom ?? [])
    .filter((metric) => metric.id?.trim() && metric.name?.trim())
    .sort((a, b) =>
      [a.group, a.subgroup, a.name]
        .map((part) => part?.trim() ?? '')
        .join('\u0000')
        .localeCompare(
          [b.group, b.subgroup, b.name].map((part) => part?.trim() ?? '').join('\u0000'),
        ),
    )
    .map((metric) => {
      const status = (metric.status || 'unknown').trim().toLowerCase();
      const hasValue = typeof metric.value === 'number' && Number.isFinite(metric.value);
      const kind = (metric.kind || 'number').trim().toLowerCase();
      let measured = 'Unavailable';
      if (hasValue && kind === 'boolean') {
        measured = (metric.value as number) >= 0.5 ? 'Online' : 'Offline';
      } else if (hasValue && kind === 'timestamp') {
        measured = formatCustomSensorAge(metric.value as number);
      } else if (hasValue) {
        measured = formatCustomSensorValue(metric.value as number, metric.unit);
      }
      const value = metric.stale ? `${measured} (stale)` : measured;
      const details = [
        status === 'ok' ? 'OK' : titleCaseDelimitedLabel(status),
        metric.error?.trim(),
      ].filter(Boolean);
      const label = [metric.group, metric.subgroup, metric.name]
        .map((part) => part?.trim())
        .filter(Boolean)
        .join(' / ');
      return {
        label,
        value,
        valueTitle: details.length > 0 ? `${value} · ${details.join(' · ')}` : value,
      };
    });

export const formatInteger = (value?: number): string => {
  if (value === undefined || value === null || Number.isNaN(value)) return '—';
  return new Intl.NumberFormat().format(Math.round(value));
};