Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
import type { Alert } from '@/types/api';
import type { Resource } from '@/types/resource';
import { getActionableAgentIdFromResource } from '@/utils/agentResources';
import { isAlertsDetectionEnabled } from '@/utils/alertsActivation';
const noAlertStyles = {
rowClass: '',
indicatorClass: '',
badgeClass: '',
hasAlert: false,
alertCount: 0,
severity: null as 'critical' | 'warning' | 'info' | null,
hasPoweredOffAlert: false,
hasNonPoweredOffAlert: false,
hasUnacknowledgedAlert: false,
unacknowledgedCount: 0,
acknowledgedCount: 0,
hasAcknowledgedOnlyAlert: false,
};
// Get alert highlighting styles based on active alerts for a resource.
// When nodeMatch is provided, also includes alerts whose `node` field matches
// (covers storage/topology/disk alerts that belong to a node but have a
// different resourceId than the node itself).
export const getAlertStyles = (
resourceId: string | string[],
activeAlerts: Record<string, Alert>,
alertsEnabled: boolean | undefined = isAlertsDetectionEnabled(),
nodeMatch?: string,
) => {
if (!alertsEnabled) {
return noAlertStyles;
}
return getAlertStylesForAlerts(
getAlertsForResource(
Array.isArray(resourceId) ? resourceId : [resourceId],
activeAlerts,
alertsEnabled,
nodeMatch,
),
);
};
// Alert highlighting for a unified resource's row, from the open alerts its
// drawer lists under "Needs attention" (getAlertsForUnifiedResource). Matching
// by display name misses machines whose alerts are keyed on the agent or
// Docker host id, and a row that disagrees with its own drawer hides the
// problem until someone opens it.
export const getUnifiedResourceAlertStyles = (
resource: Resource,
activeAlerts: Record<string, Alert>,
alertsEnabled: boolean | undefined = isAlertsDetectionEnabled(),
) => {
if (!alertsEnabled) {
return noAlertStyles;
}
return getAlertStylesForAlerts(
getAlertsForUnifiedResource(resource, activeAlerts, alertsEnabled),
);
};
function getAlertStylesForAlerts(alertsForResource: Alert[]) {
const unacknowledgedAlerts = alertsForResource.filter((alert) => !alert.acknowledged);
const acknowledgedAlerts = alertsForResource.filter((alert) => alert.acknowledged);
let highestSeverity: 'critical' | 'warning' | 'info' | null = null;
let hasPoweredOffAlert = false;
let hasNonPoweredOffAlert = false;
unacknowledgedAlerts.forEach((alert) => {
if (
alert.level === 'critical' ||
(alert.level === 'warning' && highestSeverity !== 'critical') ||
(alert.level === 'info' && highestSeverity === null)
) {
highestSeverity = alert.level;
}
if (alert.type === 'powered-off') {
hasPoweredOffAlert = true;
} else {
hasNonPoweredOffAlert = true;
}
});
const alertCount = alertsForResource.length;
const unacknowledgedCount = unacknowledgedAlerts.length;
const acknowledgedCount = acknowledgedAlerts.length;
const hasUnacknowledgedAlert = unacknowledgedCount > 0;
const hasAlert = alertCount > 0;
if (highestSeverity === 'critical') {
return {
rowClass: 'bg-red-50 dark:bg-red-950/25 border-l-4 border-red-500 dark:border-red-400',
indicatorClass: 'bg-red-500',
badgeClass: 'bg-red-100 text-red-800 dark:bg-red-900/25 dark:text-red-200',
hasAlert,
alertCount,
severity: 'critical' as const,
hasPoweredOffAlert,
hasNonPoweredOffAlert,
hasUnacknowledgedAlert,
unacknowledgedCount,
acknowledgedCount,
hasAcknowledgedOnlyAlert: !hasUnacknowledgedAlert && acknowledgedCount > 0,
};
}
if (highestSeverity === 'warning') {
return {
rowClass:
'bg-yellow-50 dark:bg-yellow-950/25 border-l-4 border-yellow-500 dark:border-yellow-400',
indicatorClass: 'bg-yellow-500',
badgeClass: 'bg-yellow-100 text-yellow-800 dark:bg-yellow-900/25 dark:text-yellow-200',
hasAlert,
alertCount,
severity: 'warning' as const,
hasPoweredOffAlert,
hasNonPoweredOffAlert,
hasUnacknowledgedAlert,
unacknowledgedCount,
acknowledgedCount,
hasAcknowledgedOnlyAlert: !hasUnacknowledgedAlert && acknowledgedCount > 0,
};
}
if (highestSeverity === 'info') {
return {
rowClass: 'bg-blue-50 dark:bg-blue-950/25 border-l-4 border-blue-500 dark:border-blue-400',
indicatorClass: 'bg-blue-500',
badgeClass: 'bg-blue-100 text-blue-800 dark:bg-blue-900/25 dark:text-blue-200',
hasAlert,
alertCount,
severity: 'info' as const,
hasPoweredOffAlert,
hasNonPoweredOffAlert,
hasUnacknowledgedAlert,
unacknowledgedCount,
acknowledgedCount,
hasAcknowledgedOnlyAlert: !hasUnacknowledgedAlert && acknowledgedCount > 0,
};
}
return {
rowClass: '',
indicatorClass: '',
badgeClass: '',
hasAlert,
alertCount,
severity: null,
hasPoweredOffAlert,
hasNonPoweredOffAlert,
hasUnacknowledgedAlert,
unacknowledgedCount,
acknowledgedCount,
hasAcknowledgedOnlyAlert: !hasUnacknowledgedAlert && acknowledgedCount > 0,
};
}
export function getAlertsForResource(
resourceIds: string[],
activeAlerts: Record<string, Alert>,
alertsEnabled: boolean | undefined = isAlertsDetectionEnabled(),
nodeMatch?: string,
): Alert[] {
if (!alertsEnabled) return [];
const ids = new Set(resourceIds.filter(Boolean));
return Object.values(activeAlerts).filter(
(alert) => ids.has(alert.resourceId) || (nodeMatch !== undefined && alert.node === nodeMatch),
);
}
const ALERT_LEVEL_RANK: Record<Alert['level'], number> = { critical: 0, warning: 1, info: 2 };
// The alert keys one unified resource answers to. Alerts raised on the unified
// resource itself (provider incidents, availability, TrueNAS, vSphere,
// Kubernetes) carry its id; poller alerts carry the canonical primary id, the
// Proxmox source id or the metrics target (PBS, PMG, storage). A machine also
// answers to its agent ("agent:<id>") and Docker runtime ("docker:<id>") keys,
// and its components' alerts nest under those: "agent:<id>/disk:<mount>",
// "agent:<id>/raid:<device>", "docker:<id>/<container>".
const getUnifiedResourceAlertKeys = (
resource: Resource,
): {
exact: Set<string>;
componentPrefixes: string[];
localStorage?: { node: string; instance?: string };
} => {
const exact = new Set<string>();
const add = (value: string | undefined) => {
const trimmed = value?.trim();
if (trimmed) exact.add(trimmed);
};
add(resource.id);
add(resource.canonicalIdentity?.primaryId);
resource.canonicalIdentity?.supersededIds?.forEach(add);
add(resource.proxmox?.sourceId);
add(resource.metricsTarget?.resourceId);
// PBS and PMG alerts are keyed on the instance id (internal/alerts pmg.go,
// pbs.go), which the platform tables match too; the drawer must not depend
// on a metrics target being set to find them.
add(resource.pbs?.instanceId);
add(resource.pmg?.instanceId);
// The alert engine keys a Docker container's alerts on its host and
// container ID ("docker:<host>/<container ID>"), or on its name when it has
// no ID (internal/alerts DockerContainerResourceID).
if (resource.type === 'app-container') {
const dockerHostId = resource.docker?.hostSourceId?.trim();
const containerId = resource.docker?.containerId?.trim();
const containerName = resource.name?.trim().replace(/^\/+/, '');
if (dockerHostId && containerId) add(`docker:${dockerHostId}/${containerId}`);
else if (dockerHostId && containerName) add(`docker:${dockerHostId}/name:${containerName}`);
}
const machine = new Set<string>();
const componentPrefixes: string[] = [];
let localStorage: { node: string; instance?: string } | undefined;
if (resource.type === 'agent' || resource.type === 'docker-host') {
const agentId = getActionableAgentIdFromResource(resource);
if (agentId) machine.add(agentId.startsWith('agent:') ? agentId : `agent:${agentId}`);
// The backend writes the agent key of a machine reported through a linked
// agent (a Proxmox node, a Docker host) into the canonical aliases.
resource.canonicalIdentity?.aliases
?.filter((alias) => alias.startsWith('agent:'))
.forEach((alias) => machine.add(alias));
// Docker hosts are "docker-host" resources or agents with a Docker runtime;
// the backend keys their alerts on the Docker host's source id.
const dockerHostId =
resource.docker?.hostSourceId?.trim() ||
(resource.metricsTarget?.resourceType === 'docker-host'
? resource.metricsTarget.resourceId?.trim()
: undefined);
if (dockerHostId) machine.add(`docker:${dockerHostId}`);
// A Proxmox node's physical disks raise health and wearout alerts on
// "<instance>:<node>:disk:<device>" (ProxmoxPhysicalDiskAlertResourceID).
const instance = resource.proxmox?.instance?.trim();
const nodeName = resource.proxmox?.nodeName?.trim();
if (instance && nodeName) componentPrefixes.push(`${instance}:${nodeName}:disk:`);
// Its own storage (local, local-zfs) raises usage, offline and ZFS alerts
// on the storage id under the node's name. Storage shared across a cluster
// is recorded under "cluster", so it never matches one node.
if (nodeName) localStorage = { node: nodeName, instance };
}
machine.forEach(add);
componentPrefixes.push(...[...machine].map((key) => `${key}/`));
return { exact, componentPrefixes, localStorage };
};
const isLocalStorageAlert = (
alert: Alert,
localStorage: { node: string; instance?: string } | undefined,
): boolean => {
if (!localStorage || alert.node !== localStorage.node) return false;
const resourceType = alert.metadata?.resourceType;
if (typeof resourceType !== 'string' || resourceType.trim().toLowerCase() !== 'storage') {
return false;
}
return !localStorage.instance || !alert.instance || alert.instance === localStorage.instance;
};
/**
* The resource IDs a unified resource's own alerts are raised on: its unified
* id, the poller's source or metrics-target id, and a machine's "agent:<id>"
* and "docker:<id>" keys. Alerts on its components nest under the latter.
*/
export const getUnifiedResourceAlertResourceIds = (resource: Resource): string[] => [
...getUnifiedResourceAlertKeys(resource).exact,
];
/**
* Open alerts for one unified resource, read from the websocket's active
* alert map, most severe first. This is the canonical source for a resource
* drawer's "Needs attention" list: resources do not embed their alerts.
*/
export function getAlertsForUnifiedResource(
resource: Resource,
activeAlerts: Record<string, Alert>,
alertsEnabled: boolean | undefined = isAlertsDetectionEnabled(),
): Alert[] {
if (!alertsEnabled) return [];
const { exact, componentPrefixes, localStorage } = getUnifiedResourceAlertKeys(resource);
return Object.values(activeAlerts)
.filter(
(alert) =>
exact.has(alert.resourceId) ||
componentPrefixes.some((prefix) => alert.resourceId?.startsWith(prefix)) ||
isLocalStorageAlert(alert, localStorage),
)
.sort(
(a, b) =>
(ALERT_LEVEL_RANK[a.level] ?? 3) - (ALERT_LEVEL_RANK[b.level] ?? 3) ||
Date.parse(a.startTime) - Date.parse(b.startTime),
);
}
/**
* Alerts raised on the resource itself, not on anything nested under it (a
* Docker host's containers and services, a machine's disks). A drawer chart
* draws only these: a container's memory trigger is not its host's.
*/
export function getOwnAlertsForUnifiedResource(
resource: Resource,
activeAlerts: Record<string, Alert>,
alertsEnabled: boolean | undefined = isAlertsDetectionEnabled(),
): Alert[] {
if (!alertsEnabled) return [];
const { exact } = getUnifiedResourceAlertKeys(resource);
return Object.values(activeAlerts).filter((alert) => exact.has(alert.resourceId));
}