Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
import type { WorkloadGuest, ViewMode, WorkloadType } from '@/types/workloads';
import {
getWorkloadInfoValue,
type IODistributionStats,
type WorkloadIOEmphasis,
} from './guestRowModel';
import {
parseFilter,
parseFilterStack,
evaluateFilterStack,
splitSearchExclusions,
} from '@/utils/searchQuery';
import { normalizeSourcePlatformQueryValue } from '@/utils/sourcePlatforms';
import { OFFLINE_HEALTH_STATUSES } from '@/utils/status';
import { hasFailedAttachedAvailabilityCheck } from '@/utils/availabilityProbePresentation';
import {
isContainerWorkloadViewMode,
getWorkloadCPUPercent,
resolveWorkloadType,
workloadMatchesPlatformScope,
workloadMatchesViewMode,
} from '@/utils/workloads';
import { getWorkloadTypePresentation } from '@/utils/workloadTypePresentation';
import { getCurrentWorkloadDiskUsage } from '@/utils/workloadDiskUsage';
import { getCurrentWorkloadMemoryUsage } from '@/utils/memoryObservation';
import {
getKubernetesContextKey,
getWorkloadHostHintCandidates,
workloadHostScopeId,
workloadNodeScopeId,
resolveWorkloadHostScope,
} from './workloadTopology';
export interface FilterWorkloadsParams {
guests: WorkloadGuest[];
viewMode: ViewMode;
statusMode: string;
searchTerm: string;
selectedNode: string | null;
selectedHostHint: string | null;
selectedPlatform?: string | null;
selectedKubernetesContext: string | null;
selectedKubernetesNamespace?: string | null;
selectedCluster?: string | null;
containerRuntime?: string | null;
}
export interface WorkloadStats {
total: number;
running: number;
degraded: number;
stopped: number;
vms: number;
containers: number;
appContainers: number;
pods: number;
}
type WorkloadStatusBucket = 'running' | 'degraded' | 'stopped';
const RUNNING_HEALTH_STATUSES = new Set(['online', 'running', 'healthy']);
const classifyWorkloadStatus = (status: string): WorkloadStatusBucket => {
const normalized = status.trim().toLowerCase();
if (RUNNING_HEALTH_STATUSES.has(normalized)) return 'running';
if (OFFLINE_HEALTH_STATUSES.has(normalized)) return 'stopped';
return 'degraded';
};
const resolveWorkloadStatusBucket = (guest: WorkloadGuest): WorkloadStatusBucket => {
const resourceStatus = (guest.resourceStatus || '').trim().toLowerCase();
// Unified-resource health owns warning/offline posture, while a healthy
// resource still defers to the provider-authored runtime power state.
if (resourceStatus && !RUNNING_HEALTH_STATUSES.has(resourceStatus)) {
return classifyWorkloadStatus(resourceStatus);
}
const bucket = classifyWorkloadStatus(guest.status || '');
// A powered-on guest that its own probe cannot reach is not healthy, even
// though the hypervisor only knows it is running. A stopped guest keeps its
// own bucket; the probe says nothing useful about a guest nobody expects to
// answer.
if (bucket === 'running' && hasFailedAttachedAvailabilityCheck(guest)) {
return 'degraded';
}
// A running guest with an open warning or critical alert needs attention
// even while its power state and resource status read healthy. Backup age
// is deliberately not counted here: it has its own column and tab.
if (bucket === 'running' && guest.hasOpenAlert) {
return 'degraded';
}
return bucket;
};
type SortDirection = 'asc' | 'desc';
type SortValue = string | number | boolean | null | undefined;
export const getWorkloadSearchCandidates = (
guest: WorkloadGuest,
): Array<string | number | undefined> => [
guest.name,
guest.id,
guest.displayId,
guest.vmid,
guest.node,
guest.status,
guest.image,
guest.namespace,
guest.contextLabel,
guest.clusterName,
guest.instance,
guest.platformType,
guest.platformScopes?.join(' '),
guest.workloadType,
guest.containerRuntime,
guest.containerId,
guest.dockerHostId,
guest.dockerHostName,
guest.kubernetesAgentId,
];
const matchesWorkloadTextSearch = (guest: WorkloadGuest, term: string): boolean => {
const needle = term.trim().toLowerCase();
if (!needle) return true;
return getWorkloadSearchCandidates(guest)
.filter(
(value): value is string | number => typeof value === 'string' || typeof value === 'number',
)
.some((value) => String(value).toLowerCase().includes(needle));
};
export interface SelectVisibleWorkloadInventoryParams {
guests: WorkloadGuest[];
excludedTypes: ReadonlySet<WorkloadType>;
platformScope?: string | null;
}
// A surface's forced platform (Proxmox, vSphere) is the page's identity, not
// a user-applied filter, so it scopes the inventory itself: chip counts,
// search suggestions, and availability detection must describe the same guest
// set the table can ever render.
export const selectVisibleWorkloadInventory = ({
guests,
excludedTypes,
platformScope,
}: SelectVisibleWorkloadInventoryParams): WorkloadGuest[] => {
const normalizedScope = normalizeSourcePlatformQueryValue(platformScope);
const scopeActive = Boolean(normalizedScope && normalizedScope !== 'all');
if (excludedTypes.size === 0 && !scopeActive) return guests;
return guests.filter(
(guest) =>
!excludedTypes.has(resolveWorkloadType(guest)) &&
(!scopeActive || workloadMatchesPlatformScope(guest, normalizedScope)),
);
};
export const filterWorkloads = ({
guests: allGuests,
viewMode,
statusMode,
searchTerm,
selectedNode,
selectedHostHint,
selectedPlatform,
selectedKubernetesContext,
selectedKubernetesNamespace,
selectedCluster,
containerRuntime,
}: FilterWorkloadsParams): WorkloadGuest[] => {
let guests = allGuests;
const nodeScope = selectedNode;
if (nodeScope && viewMode !== 'pod') {
const resolvedScope = resolveWorkloadHostScope(allGuests, nodeScope);
guests = resolvedScope ? guests.filter((g) => workloadHostScopeId(g) === resolvedScope) : [];
}
const hostHint = (selectedHostHint || '').trim().toLowerCase();
if (!nodeScope && hostHint && viewMode !== 'pod') {
guests = guests.filter((g) => {
if (resolveWorkloadType(g) === 'pod') return false;
return getWorkloadHostHintCandidates(g).some((candidate) =>
candidate.toLowerCase().includes(hostHint),
);
});
}
const k8sContext = selectedKubernetesContext;
if (k8sContext && viewMode === 'pod') {
guests = guests.filter(
(g) => resolveWorkloadType(g) === 'pod' && getKubernetesContextKey(g) === k8sContext,
);
}
const k8sNamespace = (selectedKubernetesNamespace || '').trim();
if (k8sNamespace && viewMode === 'pod') {
guests = guests.filter((g) => {
if (resolveWorkloadType(g) !== 'pod') return false;
return (g.namespace || '').trim() === k8sNamespace;
});
}
const vmCluster = (selectedCluster || '').trim();
if (vmCluster && viewMode === 'vm') {
guests = guests.filter((g) => (g.clusterName || '').trim() === vmCluster);
}
if (viewMode !== 'all') {
guests = guests.filter((g) => workloadMatchesViewMode(resolveWorkloadType(g), viewMode));
}
const normalizedPlatform = normalizeSourcePlatformQueryValue(selectedPlatform);
if (normalizedPlatform && normalizedPlatform !== 'all') {
guests = guests.filter((g) => workloadMatchesPlatformScope(g, normalizedPlatform));
}
const normalizedRuntime = (containerRuntime || '').trim().toLowerCase();
if (normalizedRuntime && isContainerWorkloadViewMode(viewMode)) {
guests = guests.filter((g) => {
if (resolveWorkloadType(g) !== 'app-container') return false;
const runtime = (g.containerRuntime || '').trim().toLowerCase();
return runtime === normalizedRuntime;
});
}
if (statusMode === 'running') {
guests = guests.filter((g) => resolveWorkloadStatusBucket(g) === 'running');
} else if (statusMode === 'degraded') {
guests = guests.filter((g) => resolveWorkloadStatusBucket(g) === 'degraded');
} else if (statusMode === 'stopped') {
guests = guests.filter((g) => resolveWorkloadStatusBucket(g) === 'stopped');
}
const trimmedSearch = searchTerm.trim();
if (trimmedSearch) {
const searchParts = trimmedSearch
.split(',')
.map((t) => t.trim())
.filter((t) => t);
const filters: string[] = [];
const textSearches: string[] = [];
// `-term` exclusions apply across the whole result set (AND NOT), while
// the remaining comma-separated text parts keep their OR semantics.
const exclusions: string[] = [];
searchParts.forEach((part) => {
if (
part.includes('>') ||
part.includes('<') ||
part.includes(':') ||
parseFilter(part).type === 'metric'
) {
filters.push(part);
} else {
const split = splitSearchExclusions(part);
exclusions.push(...split.excludes);
if (split.needle) textSearches.push(split.needle);
}
});
if (filters.length > 0) {
const filterString = filters.join(' AND ');
const stack = parseFilterStack(filterString);
if (stack.filters.length > 0) {
guests = guests.filter((g) => evaluateFilterStack(g, stack));
}
}
if (exclusions.length > 0) {
guests = guests.filter((g) => !exclusions.some((term) => matchesWorkloadTextSearch(g, term)));
}
if (textSearches.length > 0) {
guests = guests.filter((g) =>
textSearches.some((term) => matchesWorkloadTextSearch(g, term)),
);
}
}
return guests;
};
export const getDiskUsagePercent = (guest: WorkloadGuest): number | null => {
// The row may retain a clearly labelled last-known number. It must not
// compete with current readings when sorting capacity, in either direction.
return getCurrentWorkloadDiskUsage(guest);
};
export const createWorkloadSortComparator = (
sortKey: string,
sortDirection: SortDirection,
options: {
memoryValue?: (guest: WorkloadGuest) => number | null | undefined;
} = {},
): ((a: WorkloadGuest, b: WorkloadGuest) => number) | null => {
if (!sortKey) {
return null;
}
const tiebreak = (a: WorkloadGuest, b: WorkloadGuest): number => {
const nameA = (a.name || '').toLowerCase();
const nameB = (b.name || '').toLowerCase();
if (nameA !== nameB) return nameA < nameB ? -1 : 1;
return a.id < b.id ? -1 : a.id > b.id ? 1 : 0;
};
const naturalTextCollator = new Intl.Collator(undefined, {
numeric: true,
sensitivity: 'base',
});
const memorySortValue = (guest: WorkloadGuest): number | null => {
const usage = getCurrentWorkloadMemoryUsage(guest);
if (usage === null) return null;
const value = options.memoryValue ? options.memoryValue(guest) : usage;
return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null;
};
return (a: WorkloadGuest, b: WorkloadGuest): number => {
let aVal: SortValue = null;
let bVal: SortValue = null;
if (sortKey === 'info') {
aVal = getWorkloadInfoValue(a);
bVal = getWorkloadInfoValue(b);
} else if (sortKey === 'cpu') {
aVal = getWorkloadCPUPercent(a.cpu) ?? 0;
bVal = getWorkloadCPUPercent(b.cpu) ?? 0;
} else if (sortKey === 'memory') {
aVal = memorySortValue(a);
bVal = memorySortValue(b);
} else if (sortKey === 'disk') {
aVal = getDiskUsagePercent(a);
bVal = getDiskUsagePercent(b);
} else if (sortKey === 'diskIo') {
aVal = Math.max(0, a.diskRead || 0) + Math.max(0, a.diskWrite || 0);
bVal = Math.max(0, b.diskRead || 0) + Math.max(0, b.diskWrite || 0);
} else if (sortKey === 'netIo') {
aVal = Math.max(0, a.networkIn || 0) + Math.max(0, a.networkOut || 0);
bVal = Math.max(0, b.networkIn || 0) + Math.max(0, b.networkOut || 0);
} else {
aVal = (a as unknown as Record<string, SortValue>)[sortKey];
bVal = (b as unknown as Record<string, SortValue>)[sortKey];
}
const aIsEmpty = aVal === null || aVal === undefined || aVal === '';
const bIsEmpty = bVal === null || bVal === undefined || bVal === '';
if (aIsEmpty && bIsEmpty) return tiebreak(a, b);
if (aIsEmpty) return 1;
if (bIsEmpty) return -1;
if (typeof aVal === 'number' && typeof bVal === 'number') {
if (aVal === bVal) return tiebreak(a, b);
const comparison = aVal < bVal ? -1 : 1;
return sortDirection === 'asc' ? comparison : -comparison;
}
const aStr = String(aVal).toLowerCase();
const bStr = String(bVal).toLowerCase();
if (aStr === bStr) return tiebreak(a, b);
const comparison =
sortKey === 'info' ? naturalTextCollator.compare(aStr, bStr) : aStr < bStr ? -1 : 1;
if (comparison === 0) return tiebreak(a, b);
return sortDirection === 'asc' ? comparison : -comparison;
};
};
export const getWorkloadGroupKey = (guest: WorkloadGuest): string => {
const type = resolveWorkloadType(guest);
if (type === 'vm' || type === 'system-container') {
const scope = workloadNodeScopeId(guest);
if (
guest.platformScopes?.includes('vmware-vsphere') &&
!guest.platformScopes.includes('proxmox-pve')
) {
// Cluster, connection and host labels can arrive after the first snapshot.
// Group ownership follows the server parent so enrichment keeps rows mounted.
const parentId = guest.parentResourceId?.trim();
return parentId ? `vsphere|parent|${encodeURIComponent(parentId)}` : `vsphere|${scope}`;
}
return scope;
}
const context = guest.contextLabel || guest.node || guest.instance || guest.namespace || guest.id;
return `${type}:${context}`;
};
export const getWorkloadGroupLabel = (
groupKey: string,
guests: WorkloadGuest[],
groupNodeName?: string | null,
groupTypeLabel?: string | null,
): { type: string; name: string } => {
const normalizedGroupKey = guests.length > 0 ? getWorkloadGroupKey(guests[0]) : groupKey;
const [prefix, ...rest] = normalizedGroupKey.split(':');
const context = rest.length > 0 ? rest.join(':') : normalizedGroupKey;
const normalizedNodeName = (groupNodeName || '').trim();
const normalizedGroupTypeLabel = (groupTypeLabel || '').trim();
if (normalizedNodeName) {
return { type: normalizedGroupTypeLabel, name: normalizedNodeName };
}
if (
prefix === 'app-container' ||
prefix === 'pod' ||
prefix === 'vm' ||
prefix === 'system-container'
) {
return {
type: normalizedGroupTypeLabel || getWorkloadTypePresentation(prefix).pluralLabel,
name: context,
};
}
const first = guests[0];
if (first) {
const cluster = (first.clusterName || '').trim();
const nodeName = (first.node || '').trim();
if (nodeName && cluster) return { type: cluster, name: nodeName };
if (nodeName) return { type: '', name: nodeName };
const instance = (first.instance || '').trim();
if (instance) return { type: '', name: instance };
}
return { type: '', name: context };
};
export const groupWorkloads = (
guests: WorkloadGuest[],
mode: 'grouped' | 'flat',
comparator: ((a: WorkloadGuest, b: WorkloadGuest) => number) | null,
): Record<string, WorkloadGuest[]> => {
if (mode === 'flat') {
const groups: Record<string, WorkloadGuest[]> = { '': guests };
if (comparator) {
groups[''] = groups[''].sort(comparator);
}
return groups;
}
const groups: Record<string, WorkloadGuest[]> = {};
guests.forEach((guest) => {
const nodeId = getWorkloadGroupKey(guest);
if (!groups[nodeId]) {
groups[nodeId] = [];
}
groups[nodeId].push(guest);
});
if (comparator) {
Object.keys(groups).forEach((nodeId) => {
groups[nodeId] = groups[nodeId].sort(comparator);
});
}
return groups;
};
export const computeWorkloadStats = (guests: WorkloadGuest[]): WorkloadStats => {
let running = 0;
let degraded = 0;
let stopped = 0;
guests.forEach((guest) => {
const bucket = resolveWorkloadStatusBucket(guest);
if (bucket === 'running') running += 1;
else if (bucket === 'degraded') degraded += 1;
else stopped += 1;
});
const vms = guests.filter((g) => resolveWorkloadType(g) === 'vm').length;
const containers = guests.filter((g) => resolveWorkloadType(g) === 'system-container').length;
const appContainers = guests.filter((g) => resolveWorkloadType(g) === 'app-container').length;
const pods = guests.filter((g) => resolveWorkloadType(g) === 'pod').length;
return {
total: guests.length,
running,
degraded,
stopped,
vms,
containers,
appContainers,
pods,
};
};
const computeMedian = (values: number[]): number => {
if (values.length === 0) return 0;
const sorted = [...values].sort((a, b) => a - b);
const mid = Math.floor(sorted.length / 2);
if (sorted.length % 2 === 0) {
return (sorted[mid - 1] + sorted[mid]) / 2;
}
return sorted[mid];
};
const computePercentile = (values: number[], percentile: number): number => {
if (values.length === 0) return 0;
const sorted = [...values].sort((a, b) => a - b);
const clamped = Math.max(0, Math.min(1, percentile));
const index = Math.max(0, Math.min(sorted.length - 1, Math.ceil(clamped * sorted.length) - 1));
return sorted[index];
};
// Median absolute deviation plus upper percentiles feed getOutlierEmphasis in
// guestRowModel; non-finite and negative samples are dropped.
const buildIODistribution = (values: number[]): IODistributionStats => {
const valid = values.filter((value) => Number.isFinite(value) && value >= 0);
if (valid.length === 0) {
return { median: 0, mad: 0, max: 0, p97: 0, p99: 0, count: 0 };
}
const median = computeMedian(valid);
const deviations = valid.map((value) => Math.abs(value - median));
const mad = computeMedian(deviations);
const max = Math.max(...valid, 0);
const p97 = computePercentile(valid, 0.97);
const p99 = computePercentile(valid, 0.99);
return { median, mad, max, p97, p99, count: valid.length };
};
export const computeWorkloadIOEmphasis = (guests: WorkloadGuest[]): WorkloadIOEmphasis => ({
network: buildIODistribution(
guests.map((guest) => Math.max(0, guest.networkIn ?? 0) + Math.max(0, guest.networkOut ?? 0)),
),
diskIO: buildIODistribution(
guests.map((guest) => Math.max(0, guest.diskRead ?? 0) + Math.max(0, guest.diskWrite ?? 0)),
),
});