Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

10.7 KB
Raw Blame
import type { ResourceType as MetricsHistoryResourceType } from '@/api/charts';
import {
  GPU_METRICS_HISTORY_GROUPS,
  HOST_METRICS_HISTORY_GROUPS,
} from '@/components/shared/hostMetricsHistoryModel';
import type {
  GuestDrawerHistoryDeferredMetric,
  GuestDrawerHistoryTarget,
} from '@/components/Workloads/guestDrawerModel';
import {
  GUEST_DRAWER_HISTORY_GROUPS,
  type GuestDrawerHistoryGroupConfig,
} from '@/components/Workloads/guestDrawerModel';
import type { HostGPUSensor } from '@/types/api';
import type { Resource } from '@/types/resource';
import { getDiskPercent, getMemoryPercent } from '@/types/resource';
import {
  getPhysicalDiskTemperaturePresentation,
  isPhysicalDiskTemperatureCurrent,
} from '@/features/storageBackups/diskTemperaturePresentation';
import { asTrimmedString } from '@/utils/stringUtils';
import { getMemoryObservationPresentation } from '@/utils/memoryObservation';
import { getWorkloadGuestDiskRead } from '@/utils/workloadGuestPresentation';
import { getAgentFirstReportStatusIndicator } from '@/utils/agentResources';

const finiteMetric = (value: number | undefined): number | undefined =>
  typeof value === 'number' && Number.isFinite(value) ? value : undefined;

const maxFiniteGPUValue = (
  resource: Resource,
  select: (gpu: HostGPUSensor) => number | undefined,
): number | undefined => {
  let maximum: number | undefined;
  for (const gpu of resource.agent?.sensors?.gpu ?? []) {
    const value = finiteMetric(select(gpu));
    if (value === undefined) continue;
    maximum = maximum === undefined ? value : Math.max(maximum, value);
  }
  return maximum;
};

export const getResourceGPUUtilizationPercent = (resource: Resource): number | undefined =>
  maxFiniteGPUValue(resource, (gpu) => {
    const utilization = finiteMetric(gpu.utilizationPercent);
    return utilization !== undefined && utilization >= 0 && utilization <= 100
      ? utilization
      : undefined;
  });

export const getResourceGPUMemoryPercent = (resource: Resource): number | undefined =>
  maxFiniteGPUValue(resource, (gpu) => {
    const used = finiteMetric(gpu.memoryUsedBytes);
    const total = finiteMetric(gpu.memoryTotalBytes);
    if (used === undefined || total === undefined || used < 0 || total <= 0) return undefined;
    return Math.min(100, Math.max(0, (used / total) * 100));
  });

export const getResourceGPUTemperatureCelsius = (resource: Resource): number | undefined =>
  maxFiniteGPUValue(resource, (gpu) => {
    const temperature = finiteMetric(gpu.temperatureCelsius);
    return temperature !== undefined && temperature > 0 && temperature <= 150
      ? temperature
      : undefined;
  });

const NODE_METRICS_HISTORY_GROUPS = HOST_METRICS_HISTORY_GROUPS.filter(
  (group) => group.id !== 'disk-io',
);

const STORAGE_METRICS_HISTORY_GROUPS: GuestDrawerHistoryGroupConfig[] = [
  {
    id: 'capacity',
    label: 'Capacity',
    unit: '%',
    series: [{ metric: 'usage', label: 'Used', unit: '%', color: '#22c55e' }],
  },
];

const DISK_METRICS_HISTORY_GROUPS: GuestDrawerHistoryGroupConfig[] = [
  {
    id: 'disk-activity',
    label: 'Activity',
    unit: '%',
    series: [{ metric: 'disk', label: 'Busy', unit: '%', color: '#8b5cf6' }],
  },
  {
    id: 'disk-io',
    label: 'Disk I/O',
    unit: 'B/s',
    series: [
      { metric: 'diskread', label: 'Read', unit: 'B/s', color: '#3b82f6' },
      { metric: 'diskwrite', label: 'Write', unit: 'B/s', color: '#f59e0b' },
    ],
  },
  {
    id: 'disk-thermal',
    label: 'Temperature',
    unit: 'C',
    series: [{ metric: 'smart_temp', label: 'Disk', unit: 'C', color: '#ef4444' }],
  },
];

const getBaseMetricsHistoryGroups = (
  resourceType: MetricsHistoryResourceType | undefined,
): GuestDrawerHistoryGroupConfig[] => {
  switch (resourceType) {
    case 'agent':
    case 'docker-host':
      return HOST_METRICS_HISTORY_GROUPS;
    case 'node':
      return NODE_METRICS_HISTORY_GROUPS;
    case 'storage':
      return STORAGE_METRICS_HISTORY_GROUPS;
    case 'disk':
      return DISK_METRICS_HISTORY_GROUPS;
    default:
      return GUEST_DRAWER_HISTORY_GROUPS;
  }
};

export const getResourceMetricsHistoryGroups = (
  resource: Resource,
): GuestDrawerHistoryGroupConfig[] => {
  const target = getResourceMetricsHistoryTarget(resource);
  const baseGroups = getBaseMetricsHistoryGroups(target?.resourceType);
  const supportsGPUHistory =
    target?.resourceType === 'agent' ||
    target?.resourceType === 'docker-host' ||
    target?.resourceType === 'node';
  return supportsGPUHistory && (resource.agent?.sensors?.gpu?.length ?? 0) > 0
    ? [...baseGroups, ...GPU_METRICS_HISTORY_GROUPS]
    : baseGroups;
};

export const getResourceMetricsHistoryTarget = (
  resource: Resource,
): GuestDrawerHistoryTarget | null => {
  const metricsType = asTrimmedString(resource.metricsTarget?.resourceType);
  const metricsId = asTrimmedString(resource.metricsTarget?.resourceId);
  if (metricsType && metricsId) {
    return {
      resourceType: metricsType as MetricsHistoryResourceType,
      resourceId: metricsId,
    };
  }

  if (resource.type === 'agent') {
    const resourceId = asTrimmedString(resource.id);
    return resourceId ? { resourceType: 'agent', resourceId } : null;
  }

  return null;
};

// Any resource that resolves a metrics history target can chart history —
// the backend stores and serves series for every type resolveMetricsTarget
// hands out. Each target type selects the catalog that matches the metrics
// actually persisted for it; storage and physical disks must not inherit the
// CPU/network workload catalog merely because they use the shared renderer.
// Gating on type === 'agent' silently hid history for Docker containers
// even though the store records their CPU/memory/disk/IO samples.
export const resourceSupportsMetricsHistory = (resource: Resource): boolean =>
  getResourceMetricsHistoryTarget(resource) !== null;

const isProxmoxGuest = (resource: Resource): boolean =>
  (resource.type === 'vm' || resource.type === 'system-container') &&
  (resource.platformType === 'proxmox-pve' ||
    resource.platformScopes?.includes('proxmox-pve') === true);

// The guest kind decides whether a Proxmox listing reading counts cache, the
// same distinction the Workloads drawer makes.
const memoryReading = (resource: Resource) =>
  getMemoryObservationPresentation(
    resource.memory ? finiteMetric(getMemoryPercent(resource)) : undefined,
    resource.memory?.observation,
    isProxmoxGuest(resource),
    {
      agentStale: resource.agent?.stale === true,
      guestKind:
        resource.type === 'vm'
          ? 'vm'
          : resource.type === 'system-container'
            ? 'container'
            : undefined,
    },
  );

const guestDiskReading = (resource: Resource) => {
  if (!isProxmoxGuest(resource)) return null;
  // A linked source can keep the canonical resource online while Proxmox
  // reports the VM stopped. Judge filesystem carriers by the native state,
  // using the same read admission as Workloads and Alerts, not Last seen.
  const status = (asTrimmedString(resource.proxmox?.runtimeStatus) || resource.status)
    .trim()
    .toLowerCase();
  return getWorkloadGuestDiskRead(
    {
      status: status === 'offline' ? 'stopped' : status,
      diskStatusReason: resource.proxmox?.diskStatusReason,
      // Match the Workloads projection: the linked agent owns filesystem
      // readings only when Proxmox supplied no filesystem list. Its own
      // reporting lease then qualifies them, not the unrelated QGA failure.
      disksFromAgent: resource.proxmox?.disks == null && resource.agent?.disks != null,
      agentStale: resource.agent?.stale === true,
    },
    resource.type === 'vm',
  );
};

// A physical disk's reading and its collection state both live on
// physicalDisk, so read them only together: websocket rows carry no top-level
// temperature for a disk, which only the REST mapper copies across. The
// reading may be a last known one kept while the current observation could
// not collect it, such as a disk in standby or a host agent past its
// reporting lease, and is then never current.
const resourceTemperature = (resource: Resource): number | undefined =>
  resource.physicalDisk ? resource.physicalDisk.temperature : resource.temperature;

const isResourceTemperatureCurrent = (resource: Resource): boolean =>
  isPhysicalDiskTemperatureCurrent(resource.physicalDisk?.collection);

export const getResourceMetricsHistoryDeferredMetrics = (
  resource: Resource,
): Record<string, GuestDrawerHistoryDeferredMetric> => {
  const metrics: Record<string, GuestDrawerHistoryDeferredMetric> = {};
  const memory = memoryReading(resource);
  if (memory && memory.state !== 'current') {
    metrics.memory = {
      lastKnownValue:
        memory.state === 'unavailable' ? undefined : finiteMetric(getMemoryPercent(resource)),
      ...(memory.state === 'unknown' ? { valueLabel: 'freshness unknown' as const } : {}),
      message: memory.message,
    };
  }
  const disk = guestDiskReading(resource);
  if (disk && disk.state !== 'current') {
    metrics.disk = {
      lastKnownValue:
        disk.state === 'last-known' && resource.disk
          ? finiteMetric(getDiskPercent(resource))
          : undefined,
      message: disk.message,
    };
  }
  if (!isResourceTemperatureCurrent(resource)) {
    const temperature = resourceTemperature(resource);
    const retained = getPhysicalDiskTemperaturePresentation({
      temperature: temperature ?? 0,
      collection: resource.physicalDisk?.collection,
    });
    if (retained?.title) {
      metrics.smart_temp = {
        lastKnownValue: temperature,
        message: `${retained.title}.`,
      };
    }
  }
  return metrics;
};

export const getResourceMetricsHistoryCurrentMetrics = (
  resource: Resource,
): Record<string, number | undefined> => {
  if (getAgentFirstReportStatusIndicator(resource)) return {};
  const disk = guestDiskReading(resource);
  const diskPercent =
    resource.disk && (!disk || disk.state === 'current')
      ? finiteMetric(getDiskPercent(resource))
      : undefined;
  const temperature = isResourceTemperatureCurrent(resource)
    ? finiteMetric(resourceTemperature(resource))
    : undefined;
  const memory = memoryReading(resource);
  return {
    cpu: finiteMetric(resource.cpu?.current),
    memory:
      resource.memory && (!memory || memory.state === 'current')
        ? finiteMetric(getMemoryPercent(resource))
        : undefined,
    disk: diskPercent,
    usage: diskPercent,
    netin: finiteMetric(resource.network?.rxBytes),
    netout: finiteMetric(resource.network?.txBytes),
    diskread: finiteMetric(resource.diskIO?.readRate),
    diskwrite: finiteMetric(resource.diskIO?.writeRate),
    temperature,
    smart_temp: temperature,
    gpu: getResourceGPUUtilizationPercent(resource),
    gpu_memory: getResourceGPUMemoryPercent(resource),
    gpu_temperature: getResourceGPUTemperatureCelsius(resource),
  };
};