Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
import type { AlertIntentSignal } from '@/api/alertIntentPolicies';
import type { Resource as UnifiedResource, ResourcePolicy } from '@/types/resource';
import type { PlatformTableColumnKind } from '@/features/platformPage/columnAlignment';
import { getPreferredInfrastructureDisplayName } from '@/utils/resourceIdentity';
import { getPulseRelaxedGuestTrigger, type MetricType } from '@/utils/metricThresholds';
const COLUMN_TOOLTIP_LOOKUP: Record<string, string> = {
'cpu %': 'Percent CPU utilization allowed before an alert fires.',
'memory %': 'Percent memory usage threshold for triggering alerts.',
'disk %': 'Percent disk usage threshold for this resource.',
'disk r mb/s': 'Maximum sustained disk read throughput before alerting.',
'disk w mb/s': 'Maximum sustained disk write throughput before alerting.',
'net in mb/s': 'Inbound network throughput threshold for alerts.',
'net out mb/s': 'Outbound network throughput threshold for alerts.',
'usage %': 'Storage capacity usage percentage that triggers an alert.',
'temp °c': 'CPU temperature limit for node alerts.',
'temperature °c': 'CPU temperature limit for node alerts.',
temperature: 'CPU temperature limit for node alerts.',
'disk temp °c': 'Individual disk temperature threshold for agents.',
'restart count': 'Maximum container restarts within the evaluation window.',
'restart window': 'Time window used to evaluate the restart count threshold.',
'restart window (s)': 'Time window used to evaluate the restart count threshold.',
'memory warn %': 'Warning threshold for container memory usage.',
'memory critical %': 'Critical threshold for container memory usage.',
'queue warn': 'Early warning when total mail queue exceeds this message count.',
'queue crit': 'Critical alert requiring urgent action when queue reaches this size.',
'deferred warn':
'Early warning for messages stuck in deferred queue (waiting to retry delivery).',
'deferred crit': 'Critical threshold for deferred messages indicating serious delivery problems.',
'hold warn': 'Early warning when administratively held messages exceed this count.',
'hold crit': 'Critical alert for held messages requiring immediate moderation attention.',
'oldest warn (min)': 'Early warning when oldest queued message exceeds this age in minutes.',
'oldest crit (min)': 'Critical alert when message queue age indicates delivery has stalled.',
'spam warn': 'Early warning for spam messages accumulating in quarantine.',
'spam crit': 'Critical spam quarantine level requiring urgent intervention.',
'virus warn': 'Early warning for virus-positive messages in quarantine.',
'virus crit': 'Critical virus quarantine threshold indicating potential outbreak.',
'growth warn %': 'Early warning when quarantine growth rate exceeds this percentage.',
'growth warn min': 'Minimum new messages required before growth percentage triggers warning.',
'growth crit %': 'Critical quarantine growth rate requiring immediate investigation.',
'growth crit min':
'Minimum new messages required before growth percentage triggers critical alert.',
'warning size (gib)': 'Total snapshot size in GiB that raises a warning.',
'critical size (gib)': 'Total snapshot size in GiB that raises a critical alert.',
};
export const ALERT_RESOURCE_TABLE_SLIDER_METRICS = new Set([
'cpu',
'memory',
'disk',
'temperature',
'diskTemperature',
]);
export type AlertResourceThresholdMap = Record<string, number | undefined>;
export interface AlertResourceTableResourceLike {
id: string;
name: string;
displayName?: string;
policy?: ResourcePolicy;
aiSafeSummary?: string;
rawName?: string;
type?: string;
thresholds?: AlertResourceThresholdMap;
defaults?: AlertResourceThresholdMap;
note?: string;
[key: string]: unknown;
}
export function flattenAlertResourceTableResources<T extends AlertResourceTableResourceLike>(
resources?: T[],
groupedResources?: Record<string, T[]>,
): T[] {
if (groupedResources) {
return Object.values(groupedResources).flat();
}
return resources ?? [];
}
export function hasAlertResourceTableRows<T extends AlertResourceTableResourceLike>(
resources?: T[],
groupedResources?: Record<string, T[]>,
globalDefaults?: AlertResourceThresholdMap,
): boolean {
if (flattenAlertResourceTableResources(resources, groupedResources).length > 0) {
return true;
}
if (groupedResources && Object.keys(groupedResources).length > 0) {
return true;
}
return Boolean(globalDefaults);
}
export function hasCustomAlertResourceGlobalDefaults(
globalDefaults?: AlertResourceThresholdMap,
factoryDefaults?: AlertResourceThresholdMap,
): boolean {
if (!globalDefaults || !factoryDefaults) {
return false;
}
return Object.keys(factoryDefaults).some((key) => {
const current = globalDefaults[key];
const factory = factoryDefaults[key];
return current !== undefined && !alertResourceThresholdsEquivalent(current, factory);
});
}
export function normalizeAlertResourceMetricKey(column: string): string {
const key = column.trim().toLowerCase();
const mapped = new Map<string, string>([
['cpu %', 'cpu'],
['memory %', 'memory'],
['disk %', 'disk'],
['disk r mb/s', 'diskRead'],
['disk w mb/s', 'diskWrite'],
['net in mb/s', 'networkIn'],
['net out mb/s', 'networkOut'],
['usage %', 'usage'],
['temp °c', 'temperature'],
['temperature °c', 'temperature'],
['temperature', 'temperature'],
['restart count', 'restartCount'],
['restart window', 'restartWindow'],
['restart window (s)', 'restartWindow'],
['memory warn %', 'memoryWarnPct'],
['memory critical %', 'memoryCriticalPct'],
['warning size (gib)', 'warningSizeGiB'],
['critical size (gib)', 'criticalSizeGiB'],
['disk temp °c', 'diskTemperature'],
['backup', 'backup'],
['snapshot', 'snapshot'],
]).get(key);
if (mapped) {
return mapped;
}
return key
.replace(' %', '')
.replace(' °c', '')
.replace(' mb/s', '')
.replace('disk r', 'diskRead')
.replace('disk w', 'diskWrite')
.replace('net in', 'networkIn')
.replace('net out', 'networkOut');
}
export function getAlertResourceMetricBounds(metric: string): { min: number; max: number } {
if (metric === 'temperature' || metric === 'diskTemperature') {
return { min: 1, max: 150 };
}
if (['diskRead', 'diskWrite', 'networkIn', 'networkOut'].includes(metric)) {
return { min: 0.01, max: 10000 };
}
if (['cpu', 'memory', 'disk', 'usage', 'memoryWarnPct', 'memoryCriticalPct'].includes(metric)) {
return { min: 1, max: 100 };
}
if (['warningSizeGiB', 'criticalSizeGiB'].includes(metric)) {
return { min: 0.01, max: 100000 };
}
if (metric === 'restartCount') {
return { min: 1, max: 50 };
}
if (metric === 'restartWindow') {
return { min: 1, max: 86400 };
}
return { min: 1, max: 10000 };
}
export function getAlertResourceMetricStep(metric: string): string | number {
if (['diskRead', 'diskWrite', 'networkIn', 'networkOut'].includes(metric)) {
return 'any';
}
if (['warningSizeGiB', 'criticalSizeGiB'].includes(metric)) {
return 'any';
}
return 1;
}
export function getAlertResourceEnabledDefault(metric: string): number {
if (['diskRead', 'diskWrite', 'networkIn', 'networkOut'].includes(metric)) {
return 100;
}
if (metric === 'temperature') {
return 80;
}
if (metric === 'diskTemperature') {
return 55;
}
if (metric === 'restartCount') {
return 3;
}
if (metric === 'restartWindow') {
return 300;
}
if (metric === 'memoryWarnPct') {
return 90;
}
if (metric === 'memoryCriticalPct') {
return 95;
}
return 80;
}
export const ALERT_RESOURCE_METRIC_OFF_VALUE = -1;
/**
* The alert engine disables a metric whenever its trigger is `<= 0`
* (`internal/alerts/canonical_metric.go`), so every editor surface has to read
* `0` as Off too: older builds advertised `0` as the disable value and those
* overrides are still on disk. Editors always stage the canonical `-1`; the
* save payload writes an Off global default as `0`, because the backend reads
* a negative default as unset.
*/
export function isAlertResourceMetricOff(
value: number | undefined | null,
inheritedDefault?: number,
): boolean {
const effectiveValue = value ?? inheritedDefault;
return (
typeof effectiveValue === 'number' && Number.isFinite(effectiveValue) && effectiveValue <= 0
);
}
/**
* Whether two thresholds alert the same way. Every value `<= 0` is Off, and a
* global default saves Off as `0` while the editors and factory defaults use
* `-1`, so the two compare equal.
*/
export function alertResourceThresholdsEquivalent(
left: number | undefined,
right: number | undefined,
): boolean {
return left === right || (isAlertResourceMetricOff(left) && isAlertResourceMetricOff(right));
}
/**
* Value an override editor must stage when a metric is switched back on.
* Clearing the override (`undefined`) only re-inherits the default, so it
* re-enables nothing when the inherited default is itself off; in that case the
* editor has to stage an explicit enabled threshold instead.
*/
export function resolveAlertResourceMetricEnableValue(
inheritedDefault: number | undefined,
metric: string,
): number | undefined {
if (inheritedDefault === undefined || isAlertResourceMetricOff(inheritedDefault)) {
return getAlertResourceEnabledDefault(metric);
}
return undefined;
}
/**
* A global default that may stay unset because another setting decides it,
* such as TrueNAS disk temperature following Disk temperature by type. Unset,
* the cell is empty with `label` as its placeholder and `title` explaining
* where the value comes from. `off` says the other setting is itself off, so
* the unset cell reads Off and switching it on has to stage `enableValue`.
*/
export interface AlertResourceGlobalDefaultFallback {
label: string;
title: string;
off?: boolean;
enableValue?: number;
}
export interface AlertResourceGlobalDefaultCell {
/** Unset and decided by the fallback. */
follows: boolean;
isOff: boolean;
/** Value staged when the cell is switched back on from Off. */
enableValue: number | undefined;
}
/**
* State of one Global Defaults metric cell. An unset default is Off, as the
* alert engine reads it, unless the metric has a fallback; then it reads as
* the fallback does. Switching it on from Off returns it to unset when the
* fallback is on, and stages an explicit value when the fallback is off,
* since unset would still be off.
*/
export function resolveAlertResourceGlobalDefaultCell(
metric: string,
value: number | undefined,
fallback: AlertResourceGlobalDefaultFallback | undefined,
): AlertResourceGlobalDefaultCell {
const follows = value === undefined && fallback !== undefined;
const explicitEnableValue = fallback?.enableValue ?? getAlertResourceEnabledDefault(metric);
return {
follows,
isOff: follows ? Boolean(fallback?.off) : isAlertResourceMetricOff(value ?? 0),
enableValue: fallback && !fallback.off ? undefined : explicitEnableValue,
};
}
export function getAlertResourceMetricDelayOverride(
metricDelaySeconds: Record<string, number> | undefined,
metric: string,
): number | undefined {
const normalized = metric.trim().toLowerCase();
const value = metricDelaySeconds?.[normalized] ?? metricDelaySeconds?.[metric];
if (typeof value !== 'number' || !Number.isFinite(value)) {
return undefined;
}
return value;
}
export function getAlertResourceColumnHeaderTooltip(column: string): string | undefined {
const normalized = column.trim().toLowerCase();
return COLUMN_TOOLTIP_LOOKUP[column] ?? COLUMN_TOOLTIP_LOOKUP[normalized];
}
export function getAlertResourceColumnKind(column: string): PlatformTableColumnKind {
const metric = normalizeAlertResourceMetricKey(column);
if (metric === 'backup' || metric === 'snapshot') {
return 'badge';
}
return 'numeric-value';
}
export function getAlertResourceInitialEditMetric(columns: string[], resourceType?: string) {
return columns
.map(normalizeAlertResourceMetricKey)
.find(
(metric) =>
metric !== 'backup' &&
metric !== 'snapshot' &&
alertResourceSupportsMetric(resourceType, metric),
);
}
// The alert signal a row's delay action configures: its first threshold
// metric the resource supports, else its offline state.
export function getAlertResourcePreferredIntentSignal(
columns: readonly string[],
resourceType: string | undefined,
): AlertIntentSignal {
for (const metric of ['cpu', 'memory', 'disk', 'temperature'] as const) {
if (
columns.some((column) => normalizeAlertResourceMetricKey(column) === metric) &&
alertResourceSupportsMetric(resourceType, metric)
) {
return `metric.${metric}`;
}
}
return 'state.offline';
}
// How the delay panel names a row it may not list itself, such as a machine
// disk or sensor: the row's label and, when known, its machine or node.
export function getAlertResourceIntentTargetLabel(resource: {
name: string;
displayName?: string;
node?: string;
}): string {
const label = resource.displayName?.trim() || resource.name;
const node = resource.node?.trim();
return node && node !== label ? `${label} · ${node}` : label;
}
export function alertResourceSupportsMetric(
resourceType: string | undefined,
metric: string,
): boolean {
if (!resourceType) return true;
if (
['node', 'agent'].includes(resourceType) &&
['diskRead', 'diskWrite', 'networkIn', 'networkOut'].includes(metric)
) {
return false;
}
if (resourceType === 'pbs') {
return ['cpu', 'memory'].includes(metric);
}
if (resourceType === 'storage') {
return metric === 'usage';
}
if (resourceType === 'agentSensor') {
return metric === 'temperature';
}
if (resourceType === 'kubernetesNamespace') {
return false;
}
if (
resourceType === 'kubernetesCluster' ||
resourceType === 'kubernetesDeployment' ||
resourceType === 'kubernetesPod'
) {
return ['cpu', 'memory', 'disk', 'diskRead', 'diskWrite', 'networkIn', 'networkOut'].includes(
metric,
);
}
if (resourceType === 'kubernetesNode') {
return ['cpu', 'memory', 'disk'].includes(metric);
}
if (resourceType === 'truenasSystem') {
return [
'cpu',
'memory',
'disk',
'diskRead',
'diskWrite',
'networkIn',
'networkOut',
'temperature',
].includes(metric);
}
if (resourceType === 'truenasPool' || resourceType === 'truenasDataset') {
return metric === 'usage';
}
if (resourceType === 'truenasDisk') {
return metric === 'temperature';
}
if (resourceType === 'vmwareHost') {
return ['cpu', 'memory', 'diskRead', 'diskWrite', 'networkIn', 'networkOut'].includes(metric);
}
if (resourceType === 'vmwareVm') {
return ['cpu', 'memory', 'disk', 'diskRead', 'diskWrite', 'networkIn', 'networkOut'].includes(
metric,
);
}
if (resourceType === 'vmwareDatastore') {
return metric === 'usage';
}
if (resourceType === 'vmwareNetwork') {
return false;
}
// A per-resource override stores only the metrics in the backend's
// ThresholdConfig. Restart and memory-limit rules exist only as Docker
// defaults, and PMG thresholds only as PMG defaults, so the backend dropped
// per-container and per-server edits of them on save.
if (resourceType === 'dockerContainer') {
return ['cpu', 'memory'].includes(metric);
}
if (resourceType === 'pmg') {
return false;
}
return true;
}
export function getAlertResourceLabel(resource: AlertResourceTableResourceLike): string {
// Operator-local /alerts table; redaction is a transmission-boundary policy
// (docs/PRIVACY.md). Use the raw infra display name to match /infrastructure.
return getPreferredInfrastructureDisplayName(resource as unknown as UnifiedResource);
}
function parseAlertMetricNumber(value: unknown): number | undefined {
if (value === undefined || value === null) return undefined;
if (typeof value === 'number') return value;
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : undefined;
}
export function getAlertResourceMetricDisplayValue(
resource: AlertResourceTableResourceLike,
metric: string,
editingThresholds?: AlertResourceThresholdMap,
isEditing = false,
): number {
const extract = (source: Record<string, unknown> | undefined) =>
parseAlertMetricNumber(source?.[metric]);
const defaults = resource.defaults as Record<string, unknown> | undefined;
if (isEditing) {
const edited = extract(editingThresholds as Record<string, unknown> | undefined);
if (edited !== undefined) {
return edited;
}
const fallback = extract(defaults);
return fallback !== undefined ? fallback : 0;
}
const liveValue = extract(resource.thresholds as Record<string, unknown> | undefined);
if (liveValue !== undefined) {
return liveValue;
}
const fallback = extract(defaults);
return fallback !== undefined ? fallback : 0;
}
const RELAXED_TRIGGER_METRIC_LABELS: Record<MetricType, string> = {
cpu: 'CPU',
memory: 'Memory',
disk: 'Disk',
};
export interface AlertResourceRelaxedTriggerSummary {
/** Raised thresholds in metric order, e.g. "CPU 95%". */
raised: string[];
title: string;
}
/**
* The alert thresholds a guest's pulse-relaxed tag raises above the values
* its row shows, as the alert engine applies them, or null when the tag
* raises none. Reads the values being edited, so the line follows an edit
* before it is saved.
*/
export function getAlertResourceRelaxedTriggerSummary(
resource: AlertResourceTableResourceLike,
editingThresholds?: AlertResourceThresholdMap,
isEditing = false,
): AlertResourceRelaxedTriggerSummary | null {
if (resource.pulseRelaxed !== true || resource.disabled === true) {
return null;
}
const rowThresholds = isEditing ? editingThresholds : resource.thresholds;
const raised = (['cpu', 'memory', 'disk'] as const).flatMap((metric) => {
const configured =
parseAlertMetricNumber(rowThresholds?.[metric]) ??
parseAlertMetricNumber(resource.defaults?.[metric]);
const trigger = getPulseRelaxedGuestTrigger(metric, configured);
if (trigger === null || (configured !== undefined && trigger <= configured)) {
return [];
}
return [`${RELAXED_TRIGGER_METRIC_LABELS[metric]} ${trigger}%`];
});
if (raised.length === 0) {
return null;
}
return {
raised,
title:
'This guest is tagged pulse-relaxed in Proxmox, which raises these alert thresholds ' +
'above the values in this row. Remove the tag in Proxmox to alert at the row values.',
};
}
export function isAlertResourceMetricOverridden(
resource: AlertResourceTableResourceLike,
metric: string,
): boolean {
return resource.thresholds?.[metric] !== undefined && resource.thresholds?.[metric] !== null;
}
export function buildAlertResourceEditPayload(resource: AlertResourceTableResourceLike) {
return {
thresholds: resource.thresholds ? { ...resource.thresholds } : {},
defaults: resource.defaults ? { ...resource.defaults } : {},
note: typeof resource.note === 'string' ? resource.note : undefined,
};
}