Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

17.8 KB
Raw Blame
import type { WorkloadGuest, ViewMode, WorkloadType } from '@/types/workloads';
import {
  getWorkloadInfoValue,
  type IODistributionStats,
  type WorkloadIOEmphasis,
} from './guestRowModel';
import {
  parseFilter,
  parseFilterStack,
  evaluateFilterStack,
  splitSearchExclusions,
} from '@/utils/searchQuery';
import { normalizeSourcePlatformQueryValue } from '@/utils/sourcePlatforms';
import { OFFLINE_HEALTH_STATUSES } from '@/utils/status';
import { hasFailedAttachedAvailabilityCheck } from '@/utils/availabilityProbePresentation';
import {
  isContainerWorkloadViewMode,
  getWorkloadCPUPercent,
  resolveWorkloadType,
  workloadMatchesPlatformScope,
  workloadMatchesViewMode,
} from '@/utils/workloads';
import { getWorkloadTypePresentation } from '@/utils/workloadTypePresentation';
import { getCurrentWorkloadDiskUsage } from '@/utils/workloadDiskUsage';
import { getCurrentWorkloadMemoryUsage } from '@/utils/memoryObservation';
import {
  getKubernetesContextKey,
  getWorkloadHostHintCandidates,
  workloadHostScopeId,
  workloadNodeScopeId,
  resolveWorkloadHostScope,
} from './workloadTopology';

export interface FilterWorkloadsParams {
  guests: WorkloadGuest[];
  viewMode: ViewMode;
  statusMode: string;
  searchTerm: string;
  selectedNode: string | null;
  selectedHostHint: string | null;
  selectedPlatform?: string | null;
  selectedKubernetesContext: string | null;
  selectedKubernetesNamespace?: string | null;
  selectedCluster?: string | null;
  containerRuntime?: string | null;
}

export interface WorkloadStats {
  total: number;
  running: number;
  degraded: number;
  stopped: number;
  vms: number;
  containers: number;
  appContainers: number;
  pods: number;
}

type WorkloadStatusBucket = 'running' | 'degraded' | 'stopped';

const RUNNING_HEALTH_STATUSES = new Set(['online', 'running', 'healthy']);

const classifyWorkloadStatus = (status: string): WorkloadStatusBucket => {
  const normalized = status.trim().toLowerCase();
  if (RUNNING_HEALTH_STATUSES.has(normalized)) return 'running';
  if (OFFLINE_HEALTH_STATUSES.has(normalized)) return 'stopped';
  return 'degraded';
};

const resolveWorkloadStatusBucket = (guest: WorkloadGuest): WorkloadStatusBucket => {
  const resourceStatus = (guest.resourceStatus || '').trim().toLowerCase();

  // Unified-resource health owns warning/offline posture, while a healthy
  // resource still defers to the provider-authored runtime power state.
  if (resourceStatus && !RUNNING_HEALTH_STATUSES.has(resourceStatus)) {
    return classifyWorkloadStatus(resourceStatus);
  }

  const bucket = classifyWorkloadStatus(guest.status || '');

  // A powered-on guest that its own probe cannot reach is not healthy, even
  // though the hypervisor only knows it is running. A stopped guest keeps its
  // own bucket; the probe says nothing useful about a guest nobody expects to
  // answer.
  if (bucket === 'running' && hasFailedAttachedAvailabilityCheck(guest)) {
    return 'degraded';
  }

  // A running guest with an open warning or critical alert needs attention
  // even while its power state and resource status read healthy. Backup age
  // is deliberately not counted here: it has its own column and tab.
  if (bucket === 'running' && guest.hasOpenAlert) {
    return 'degraded';
  }

  return bucket;
};

type SortDirection = 'asc' | 'desc';

type SortValue = string | number | boolean | null | undefined;

export const getWorkloadSearchCandidates = (
  guest: WorkloadGuest,
): Array<string | number | undefined> => [
  guest.name,
  guest.id,
  guest.displayId,
  guest.vmid,
  guest.node,
  guest.status,
  guest.image,
  guest.namespace,
  guest.contextLabel,
  guest.clusterName,
  guest.instance,
  guest.platformType,
  guest.platformScopes?.join(' '),
  guest.workloadType,
  guest.containerRuntime,
  guest.containerId,
  guest.dockerHostId,
  guest.dockerHostName,
  guest.kubernetesAgentId,
];

const matchesWorkloadTextSearch = (guest: WorkloadGuest, term: string): boolean => {
  const needle = term.trim().toLowerCase();
  if (!needle) return true;
  return getWorkloadSearchCandidates(guest)
    .filter(
      (value): value is string | number => typeof value === 'string' || typeof value === 'number',
    )
    .some((value) => String(value).toLowerCase().includes(needle));
};

export interface SelectVisibleWorkloadInventoryParams {
  guests: WorkloadGuest[];
  excludedTypes: ReadonlySet<WorkloadType>;
  platformScope?: string | null;
}

// A surface's forced platform (Proxmox, vSphere) is the page's identity, not
// a user-applied filter, so it scopes the inventory itself: chip counts,
// search suggestions, and availability detection must describe the same guest
// set the table can ever render.
export const selectVisibleWorkloadInventory = ({
  guests,
  excludedTypes,
  platformScope,
}: SelectVisibleWorkloadInventoryParams): WorkloadGuest[] => {
  const normalizedScope = normalizeSourcePlatformQueryValue(platformScope);
  const scopeActive = Boolean(normalizedScope && normalizedScope !== 'all');
  if (excludedTypes.size === 0 && !scopeActive) return guests;
  return guests.filter(
    (guest) =>
      !excludedTypes.has(resolveWorkloadType(guest)) &&
      (!scopeActive || workloadMatchesPlatformScope(guest, normalizedScope)),
  );
};

export const filterWorkloads = ({
  guests: allGuests,
  viewMode,
  statusMode,
  searchTerm,
  selectedNode,
  selectedHostHint,
  selectedPlatform,
  selectedKubernetesContext,
  selectedKubernetesNamespace,
  selectedCluster,
  containerRuntime,
}: FilterWorkloadsParams): WorkloadGuest[] => {
  let guests = allGuests;

  const nodeScope = selectedNode;
  if (nodeScope && viewMode !== 'pod') {
    const resolvedScope = resolveWorkloadHostScope(allGuests, nodeScope);
    guests = resolvedScope ? guests.filter((g) => workloadHostScopeId(g) === resolvedScope) : [];
  }

  const hostHint = (selectedHostHint || '').trim().toLowerCase();
  if (!nodeScope && hostHint && viewMode !== 'pod') {
    guests = guests.filter((g) => {
      if (resolveWorkloadType(g) === 'pod') return false;
      return getWorkloadHostHintCandidates(g).some((candidate) =>
        candidate.toLowerCase().includes(hostHint),
      );
    });
  }

  const k8sContext = selectedKubernetesContext;
  if (k8sContext && viewMode === 'pod') {
    guests = guests.filter(
      (g) => resolveWorkloadType(g) === 'pod' && getKubernetesContextKey(g) === k8sContext,
    );
  }

  const k8sNamespace = (selectedKubernetesNamespace || '').trim();
  if (k8sNamespace && viewMode === 'pod') {
    guests = guests.filter((g) => {
      if (resolveWorkloadType(g) !== 'pod') return false;
      return (g.namespace || '').trim() === k8sNamespace;
    });
  }

  const vmCluster = (selectedCluster || '').trim();
  if (vmCluster && viewMode === 'vm') {
    guests = guests.filter((g) => (g.clusterName || '').trim() === vmCluster);
  }

  if (viewMode !== 'all') {
    guests = guests.filter((g) => workloadMatchesViewMode(resolveWorkloadType(g), viewMode));
  }

  const normalizedPlatform = normalizeSourcePlatformQueryValue(selectedPlatform);
  if (normalizedPlatform && normalizedPlatform !== 'all') {
    guests = guests.filter((g) => workloadMatchesPlatformScope(g, normalizedPlatform));
  }

  const normalizedRuntime = (containerRuntime || '').trim().toLowerCase();
  if (normalizedRuntime && isContainerWorkloadViewMode(viewMode)) {
    guests = guests.filter((g) => {
      if (resolveWorkloadType(g) !== 'app-container') return false;
      const runtime = (g.containerRuntime || '').trim().toLowerCase();
      return runtime === normalizedRuntime;
    });
  }

  if (statusMode === 'running') {
    guests = guests.filter((g) => resolveWorkloadStatusBucket(g) === 'running');
  } else if (statusMode === 'degraded') {
    guests = guests.filter((g) => resolveWorkloadStatusBucket(g) === 'degraded');
  } else if (statusMode === 'stopped') {
    guests = guests.filter((g) => resolveWorkloadStatusBucket(g) === 'stopped');
  }

  const trimmedSearch = searchTerm.trim();
  if (trimmedSearch) {
    const searchParts = trimmedSearch
      .split(',')
      .map((t) => t.trim())
      .filter((t) => t);

    const filters: string[] = [];
    const textSearches: string[] = [];
    // `-term` exclusions apply across the whole result set (AND NOT), while
    // the remaining comma-separated text parts keep their OR semantics.
    const exclusions: string[] = [];

    searchParts.forEach((part) => {
      if (
        part.includes('>') ||
        part.includes('<') ||
        part.includes(':') ||
        parseFilter(part).type === 'metric'
      ) {
        filters.push(part);
      } else {
        const split = splitSearchExclusions(part);
        exclusions.push(...split.excludes);
        if (split.needle) textSearches.push(split.needle);
      }
    });

    if (filters.length > 0) {
      const filterString = filters.join(' AND ');
      const stack = parseFilterStack(filterString);
      if (stack.filters.length > 0) {
        guests = guests.filter((g) => evaluateFilterStack(g, stack));
      }
    }

    if (exclusions.length > 0) {
      guests = guests.filter((g) => !exclusions.some((term) => matchesWorkloadTextSearch(g, term)));
    }

    if (textSearches.length > 0) {
      guests = guests.filter((g) =>
        textSearches.some((term) => matchesWorkloadTextSearch(g, term)),
      );
    }
  }

  return guests;
};

export const getDiskUsagePercent = (guest: WorkloadGuest): number | null => {
  // The row may retain a clearly labelled last-known number. It must not
  // compete with current readings when sorting capacity, in either direction.
  return getCurrentWorkloadDiskUsage(guest);
};

export const createWorkloadSortComparator = (
  sortKey: string,
  sortDirection: SortDirection,
  options: {
    memoryValue?: (guest: WorkloadGuest) => number | null | undefined;
  } = {},
): ((a: WorkloadGuest, b: WorkloadGuest) => number) | null => {
  if (!sortKey) {
    return null;
  }

  const tiebreak = (a: WorkloadGuest, b: WorkloadGuest): number => {
    const nameA = (a.name || '').toLowerCase();
    const nameB = (b.name || '').toLowerCase();
    if (nameA !== nameB) return nameA < nameB ? -1 : 1;
    return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
  };
  const naturalTextCollator = new Intl.Collator(undefined, {
    numeric: true,
    sensitivity: 'base',
  });
  const memorySortValue = (guest: WorkloadGuest): number | null => {
    const usage = getCurrentWorkloadMemoryUsage(guest);
    if (usage === null) return null;
    const value = options.memoryValue ? options.memoryValue(guest) : usage;
    return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null;
  };

  return (a: WorkloadGuest, b: WorkloadGuest): number => {
    let aVal: SortValue = null;
    let bVal: SortValue = null;

    if (sortKey === 'info') {
      aVal = getWorkloadInfoValue(a);
      bVal = getWorkloadInfoValue(b);
    } else if (sortKey === 'cpu') {
      aVal = getWorkloadCPUPercent(a.cpu) ?? 0;
      bVal = getWorkloadCPUPercent(b.cpu) ?? 0;
    } else if (sortKey === 'memory') {
      aVal = memorySortValue(a);
      bVal = memorySortValue(b);
    } else if (sortKey === 'disk') {
      aVal = getDiskUsagePercent(a);
      bVal = getDiskUsagePercent(b);
    } else if (sortKey === 'diskIo') {
      aVal = Math.max(0, a.diskRead || 0) + Math.max(0, a.diskWrite || 0);
      bVal = Math.max(0, b.diskRead || 0) + Math.max(0, b.diskWrite || 0);
    } else if (sortKey === 'netIo') {
      aVal = Math.max(0, a.networkIn || 0) + Math.max(0, a.networkOut || 0);
      bVal = Math.max(0, b.networkIn || 0) + Math.max(0, b.networkOut || 0);
    } else {
      aVal = (a as unknown as Record<string, SortValue>)[sortKey];
      bVal = (b as unknown as Record<string, SortValue>)[sortKey];
    }

    const aIsEmpty = aVal === null || aVal === undefined || aVal === '';
    const bIsEmpty = bVal === null || bVal === undefined || bVal === '';

    if (aIsEmpty && bIsEmpty) return tiebreak(a, b);
    if (aIsEmpty) return 1;
    if (bIsEmpty) return -1;

    if (typeof aVal === 'number' && typeof bVal === 'number') {
      if (aVal === bVal) return tiebreak(a, b);
      const comparison = aVal < bVal ? -1 : 1;
      return sortDirection === 'asc' ? comparison : -comparison;
    }

    const aStr = String(aVal).toLowerCase();
    const bStr = String(bVal).toLowerCase();

    if (aStr === bStr) return tiebreak(a, b);
    const comparison =
      sortKey === 'info' ? naturalTextCollator.compare(aStr, bStr) : aStr < bStr ? -1 : 1;
    if (comparison === 0) return tiebreak(a, b);
    return sortDirection === 'asc' ? comparison : -comparison;
  };
};

export const getWorkloadGroupKey = (guest: WorkloadGuest): string => {
  const type = resolveWorkloadType(guest);
  if (type === 'vm' || type === 'system-container') {
    const scope = workloadNodeScopeId(guest);
    if (
      guest.platformScopes?.includes('vmware-vsphere') &&
      !guest.platformScopes.includes('proxmox-pve')
    ) {
      // Cluster, connection and host labels can arrive after the first snapshot.
      // Group ownership follows the server parent so enrichment keeps rows mounted.
      const parentId = guest.parentResourceId?.trim();
      return parentId ? `vsphere|parent|${encodeURIComponent(parentId)}` : `vsphere|${scope}`;
    }
    return scope;
  }
  const context = guest.contextLabel || guest.node || guest.instance || guest.namespace || guest.id;
  return `${type}:${context}`;
};

export const getWorkloadGroupLabel = (
  groupKey: string,
  guests: WorkloadGuest[],
  groupNodeName?: string | null,
  groupTypeLabel?: string | null,
): { type: string; name: string } => {
  const normalizedGroupKey = guests.length > 0 ? getWorkloadGroupKey(guests[0]) : groupKey;
  const [prefix, ...rest] = normalizedGroupKey.split(':');
  const context = rest.length > 0 ? rest.join(':') : normalizedGroupKey;
  const normalizedNodeName = (groupNodeName || '').trim();
  const normalizedGroupTypeLabel = (groupTypeLabel || '').trim();
  if (normalizedNodeName) {
    return { type: normalizedGroupTypeLabel, name: normalizedNodeName };
  }
  if (
    prefix === 'app-container' ||
    prefix === 'pod' ||
    prefix === 'vm' ||
    prefix === 'system-container'
  ) {
    return {
      type: normalizedGroupTypeLabel || getWorkloadTypePresentation(prefix).pluralLabel,
      name: context,
    };
  }
  const first = guests[0];
  if (first) {
    const cluster = (first.clusterName || '').trim();
    const nodeName = (first.node || '').trim();
    if (nodeName && cluster) return { type: cluster, name: nodeName };
    if (nodeName) return { type: '', name: nodeName };
    const instance = (first.instance || '').trim();
    if (instance) return { type: '', name: instance };
  }
  return { type: '', name: context };
};

export const groupWorkloads = (
  guests: WorkloadGuest[],
  mode: 'grouped' | 'flat',
  comparator: ((a: WorkloadGuest, b: WorkloadGuest) => number) | null,
): Record<string, WorkloadGuest[]> => {
  if (mode === 'flat') {
    const groups: Record<string, WorkloadGuest[]> = { '': guests };
    if (comparator) {
      groups[''] = groups[''].sort(comparator);
    }
    return groups;
  }

  const groups: Record<string, WorkloadGuest[]> = {};
  guests.forEach((guest) => {
    const nodeId = getWorkloadGroupKey(guest);
    if (!groups[nodeId]) {
      groups[nodeId] = [];
    }
    groups[nodeId].push(guest);
  });

  if (comparator) {
    Object.keys(groups).forEach((nodeId) => {
      groups[nodeId] = groups[nodeId].sort(comparator);
    });
  }

  return groups;
};

export const computeWorkloadStats = (guests: WorkloadGuest[]): WorkloadStats => {
  let running = 0;
  let degraded = 0;
  let stopped = 0;
  guests.forEach((guest) => {
    const bucket = resolveWorkloadStatusBucket(guest);
    if (bucket === 'running') running += 1;
    else if (bucket === 'degraded') degraded += 1;
    else stopped += 1;
  });
  const vms = guests.filter((g) => resolveWorkloadType(g) === 'vm').length;
  const containers = guests.filter((g) => resolveWorkloadType(g) === 'system-container').length;
  const appContainers = guests.filter((g) => resolveWorkloadType(g) === 'app-container').length;
  const pods = guests.filter((g) => resolveWorkloadType(g) === 'pod').length;

  return {
    total: guests.length,
    running,
    degraded,
    stopped,
    vms,
    containers,
    appContainers,
    pods,
  };
};

const computeMedian = (values: number[]): number => {
  if (values.length === 0) return 0;
  const sorted = [...values].sort((a, b) => a - b);
  const mid = Math.floor(sorted.length / 2);
  if (sorted.length % 2 === 0) {
    return (sorted[mid - 1] + sorted[mid]) / 2;
  }
  return sorted[mid];
};

const computePercentile = (values: number[], percentile: number): number => {
  if (values.length === 0) return 0;
  const sorted = [...values].sort((a, b) => a - b);
  const clamped = Math.max(0, Math.min(1, percentile));
  const index = Math.max(0, Math.min(sorted.length - 1, Math.ceil(clamped * sorted.length) - 1));
  return sorted[index];
};

// Median absolute deviation plus upper percentiles feed getOutlierEmphasis in
// guestRowModel; non-finite and negative samples are dropped.
const buildIODistribution = (values: number[]): IODistributionStats => {
  const valid = values.filter((value) => Number.isFinite(value) && value >= 0);
  if (valid.length === 0) {
    return { median: 0, mad: 0, max: 0, p97: 0, p99: 0, count: 0 };
  }

  const median = computeMedian(valid);
  const deviations = valid.map((value) => Math.abs(value - median));
  const mad = computeMedian(deviations);
  const max = Math.max(...valid, 0);
  const p97 = computePercentile(valid, 0.97);
  const p99 = computePercentile(valid, 0.99);

  return { median, mad, max, p97, p99, count: valid.length };
};

export const computeWorkloadIOEmphasis = (guests: WorkloadGuest[]): WorkloadIOEmphasis => ({
  network: buildIODistribution(
    guests.map((guest) => Math.max(0, guest.networkIn ?? 0) + Math.max(0, guest.networkOut ?? 0)),
  ),
  diskIO: buildIODistribution(
    guests.map((guest) => Math.max(0, guest.diskRead ?? 0) + Math.max(0, guest.diskWrite ?? 0)),
  ),
});